00000800:00000200:0.1F:1761227263.218603:0:2225:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [00000000dc627b5f] -> 12345-192.168.201.104@tcp (2) 00000800:00000200:2.1F:1761227263.227108:0:2215:0:(socklnd.c:450:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:2.1:1761227263.227133:0:2215:0:(socklnd_cb.c:753:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.104@tcp ip 192.168.201.104:988 00000800:00000200:2.1:1761227263.227138:0:2215:0:(socklnd_cb.c:772:ksocknal_queue_tx_locked()) Packet 000000008f5de369 type 2, nob 96 niov 1 nkiov 0 00000800:00000200:3.1F:1761227263.244326:0:2215:0:(socklnd.c:450:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:1.1F:1761227263.253753:0:2215:0:(socklnd.c:450:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected e ksnr_max_conns 1 00000080:00200000:2.0:1761231219.808744:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=0=0x0(2) 00000080:00200000:2.0:1761231219.808748:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231219.808752:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.808759:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.808760:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000dd2129b9, osc lock 00000000753cbb65, flags 0x400001002 00000008:00010000:2.0:1761231219.808766:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.808767:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000006f8a84e0, osc lock 0000000016d67cbc, flags 0x400001002 00010000:00010000:2.0:1761231219.808773:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.808781:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.808788:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.808794:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.808803:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.808810:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.808815:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.808819:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.808844:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 1890784 00000020:00010000:2.0:1761231219.808846:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.808848:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.808850:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000dd2129b9 (0000000001a1b4c1/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.808852:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000006f8a84e0 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.808854:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.808856:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000dd2129b9 (00000000c270dd85/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.808860:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000006f8a84e0 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.808866:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.808882:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=1431547=0x15d7fb(0) 00000080:00200008:2.0:1761231219.808888:0:11662:0:(file.c:2028:ll_file_read_iter()) file f0.fsxfile:[0x200000401:0x64f6:0x0], ppos: 1431547, count: 18560 00000080:00200000:2.0:1761231219.808894:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 349 00000080:00200000:2.0:1761231219.808932:0:11662:0:(file.c:5227:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef),name=f0.fsxfile 00000002:00010000:2.0:1761231219.808939:0:11662:0:(mdc_locks.c:1328:mdc_intent_lock()) (name: ,[0x200000401:0x64f6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1761231219.808945:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231219.808952:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1761231219.808960:0:11662:0:(dcache.c:293:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) 00000080:00010000:2.0:1761231219.808965:0:11662:0:(llite_internal.h:1711:ll_set_lock_data()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) for lock 0x311d04a022a024ba 00000080:00010000:2.0:1761231219.808971:0:11662:0:(dcache.c:193:ll_intent_drop_lock()) releasing lock with cookie 0x311d04a022a024ba from it 0000000056434e97 00010000:00010000:2.0:1761231219.808976:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231219.808994:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1761231219.809018:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231219.809024:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.809032:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.809034:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000dd2129b9, osc lock 0000000016d67cbc, flags 0x400001002 00000008:00010000:2.0:1761231219.809040:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.809041:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000006f8a84e0, osc lock 00000000753cbb65, flags 0x400001002 00010000:00010000:2.0:1761231219.809047:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809053:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809059:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809063:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809070:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809077:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809088:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809095:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.809104:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 1890784 00000020:00010000:2.0:1761231219.809106:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.809108:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.809110:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000dd2129b9 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.809113:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000006f8a84e0 (0000000001a1b4c1/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.809115:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.809117:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000dd2129b9 (00000000c270dd85/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.809120:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000006f8a84e0 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.809125:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.809133:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=0=0x0(2) 00000080:00200000:2.0:1761231219.809136:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231219.809140:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.809147:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.809148:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000dd2129b9, osc lock 00000000753cbb65, flags 0x400001002 00000008:00010000:2.0:1761231219.809153:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.809155:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000006f8a84e0, osc lock 0000000016d67cbc, flags 0x400001002 00010000:00010000:2.0:1761231219.809160:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809168:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809177:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809191:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809202:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809211:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809219:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809226:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.809239:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 1890784 00000020:00010000:2.0:1761231219.809241:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.809244:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.809246:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000dd2129b9 (0000000001a1b4c1/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.809249:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000006f8a84e0 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.809252:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.809254:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000dd2129b9 (00000000c270dd85/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.809258:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000006f8a84e0 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.809264:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.809396:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=2215945=0x21d009(0) 00000080:00200008:2.0:1761231219.809413:0:11662:0:(file.c:2180:ll_file_write_iter()) file f0.fsxfile:[0x200000401:0x64f6:0x0], ppos: 2215945, count: 42273 00000080:00200000:2.0:1761231219.809418:0:11662:0:(file.c:1671:ll_file_io_generic()) f0.fsxfile: write ppos: 2215945, count: 42273 00000080:00200000:2.0:1761231219.809421:0:11662:0:(file.c:1543:ll_io_set_mirror()) f0.fsxfile: desiginated mirror: 0 00000020:00200000:2.0:1761231219.809425:0:11662:0:(cl_io.c:216:cl_io_rw_init()) header@000000008978f1bb[0x4, 435, [0x200000401:0x64f6:0x0] hash] 00000020:00200000:2.0:1761231219.809426:0:11662:0:(cl_io.c:216:cl_io_rw_init()) io range: 2 [2215945, 2258218) 0 0 00000080:00200000:2.0:1761231219.809428:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1761231219.809435:0:11662:0:(file.c:1736:ll_file_io_generic()) Range lock [541, 551] 00020000:00200000:2.0:1761231219.809440:0:11662:0:(lov_io.c:996:lov_io_rw_iter_init()) stripe: 2 chunk: [2215945, 2258218) 2258218, 42273 00020000:00200000:2.0:1761231219.809443:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.809454:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [541, 551] 00000008:00010000:2.0:1761231219.809462:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.809464:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000380ea056, osc lock 0000000016d67cbc, flags 0x0 00010000:00010000:2.0:1761231219.809471:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809479:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1167360 1212415) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809486:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/1,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809494:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/1,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809500:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.809508:0:11662:0:(vvp_io.c:1318:vvp_io_write_start()) f0.fsxfile: write [2215945, 2258218) 00000080:00200000:2.0:1761231219.809516:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 541 of 9 to 4087 bytes 00000080:00200000:2.0:1761231219.809542:0:11662:0:(rw26.c:916:ll_write_end()) pos 2215945, len 4087, copied 4087 00000080:00200000:2.0:1761231219.809547:0:11662:0:(rw26.c:954:ll_write_end()) page@00000000edc79ef3[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.809551:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @00000000077d4da6 17ffffc0001001 3:0 ffff9001f7bb9ba0 541 lru 00000080:00200000:2.0:1761231219.809559:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@00000000261fc696 285: 1< 0x845fed 0 - - > 2< 1167360 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 0 0 8 485752832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1761231219.809562:0:11662:0:(rw26.c:954:ll_write_end()) end page@00000000edc79ef3 00000080:00200000:2.0:1761231219.809562:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 1. 00000080:00200000:2.0:1761231219.809565:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 542 of 0 to 4096 bytes 00000080:00200000:2.0:1761231219.809578:0:11662:0:(rw26.c:916:ll_write_end()) pos 2220032, len 4096, copied 4096 00000080:00200000:2.0:1761231219.809581:0:11662:0:(rw26.c:954:ll_write_end()) page@00000000eec2d817[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.809583:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @000000007abc9790 17ffffc0001001 3:0 ffff9001f7bb9dc0 542 lru 00000080:00200000:2.0:1761231219.809589:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@000000003224e8c7 286: 1< 0x845fed 0 - - > 2< 1171456 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 0 0 8 485752832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1761231219.809592:0:11662:0:(rw26.c:954:ll_write_end()) end page@00000000eec2d817 00000080:00200000:2.0:1761231219.809593:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 2. 00000080:00200000:2.0:1761231219.809595:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 543 of 0 to 4096 bytes 00000080:00200000:2.0:1761231219.809609:0:11662:0:(rw26.c:916:ll_write_end()) pos 2224128, len 4096, copied 4096 00000080:00200000:2.0:1761231219.809615:0:11662:0:(rw26.c:954:ll_write_end()) page@0000000045b5999e[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.809619:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @000000001c6ef643 17ffffc0001001 3:0 ffff9001f7bb8000 543 lru 00000080:00200000:2.0:1761231219.809629:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@00000000901ee0b9 287: 1< 0x845fed 0 - - > 2< 1175552 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 0 0 8 485752832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1761231219.809633:0:11662:0:(rw26.c:954:ll_write_end()) end page@0000000045b5999e 00000080:00200000:2.0:1761231219.809635:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 3. 00000080:00200000:2.0:1761231219.809638:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 544 of 0 to 4096 bytes 00000080:00200000:2.0:1761231219.809655:0:11662:0:(rw26.c:916:ll_write_end()) pos 2228224, len 4096, copied 4096 00000080:00200000:2.0:1761231219.809660:0:11662:0:(rw26.c:954:ll_write_end()) page@00000000ec12bdef[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.809663:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @00000000d397c037 17ffffc0001001 3:0 ffff9001f7bb9cb0 544 lru 00000080:00200000:2.0:1761231219.809668:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@0000000000f950a5 288: 1< 0x845fed 0 - - > 2< 1179648 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 0 0 8 485752832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1761231219.809670:0:11662:0:(rw26.c:954:ll_write_end()) end page@00000000ec12bdef 00000080:00200000:2.0:1761231219.809671:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 4. 00000080:00200000:2.0:1761231219.809672:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 545 of 0 to 4096 bytes 00000080:00200000:2.0:1761231219.809687:0:11662:0:(rw26.c:916:ll_write_end()) pos 2232320, len 4096, copied 4096 00000080:00200000:2.0:1761231219.809690:0:11662:0:(rw26.c:954:ll_write_end()) page@0000000080793c9a[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.809692:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @0000000079b65b26 17ffffc0001001 3:0 ffff9001f7bb8ee0 545 lru 00000080:00200000:2.0:1761231219.809696:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@00000000e738210c 289: 1< 0x845fed 0 - - > 2< 1183744 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 0 0 8 485752832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1761231219.809697:0:11662:0:(rw26.c:954:ll_write_end()) end page@0000000080793c9a 00000080:00200000:2.0:1761231219.809698:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 5. 00000080:00200000:2.0:1761231219.809699:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 546 of 0 to 4096 bytes 00000080:00200000:2.0:1761231219.809707:0:11662:0:(rw26.c:916:ll_write_end()) pos 2236416, len 4096, copied 4096 00000080:00200000:2.0:1761231219.809714:0:11662:0:(rw26.c:954:ll_write_end()) page@000000007fdc1e53[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.809718:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @0000000030584816 17ffffc0001001 3:0 ffff9001f7bb9980 546 lru 00000080:00200000:2.0:1761231219.809722:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@00000000605f333a 290: 1< 0x845fed 0 - - > 2< 1187840 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 0 0 8 485752832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1761231219.809724:0:11662:0:(rw26.c:954:ll_write_end()) end page@000000007fdc1e53 00000080:00200000:2.0:1761231219.809725:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 6. 00000080:00200000:2.0:1761231219.809726:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 547 of 0 to 4096 bytes 00000080:00200000:2.0:1761231219.809733:0:11662:0:(rw26.c:916:ll_write_end()) pos 2240512, len 4096, copied 4096 00000080:00200000:2.0:1761231219.809736:0:11662:0:(rw26.c:954:ll_write_end()) page@0000000034c2a9c9[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.809738:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @0000000029cebd49 17ffffc0001001 3:0 ffff9001f7bb8ff0 547 lru 00000080:00200000:2.0:1761231219.809742:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@0000000016203cea 291: 1< 0x845fed 0 - - > 2< 1191936 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 0 0 8 485752832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1761231219.809744:0:11662:0:(rw26.c:954:ll_write_end()) end page@0000000034c2a9c9 00000080:00200000:2.0:1761231219.809745:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 7. 00000080:00200000:2.0:1761231219.809746:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 548 of 0 to 4096 bytes 00000080:00200000:2.0:1761231219.809753:0:11662:0:(rw26.c:916:ll_write_end()) pos 2244608, len 4096, copied 4096 00000080:00200000:2.0:1761231219.809756:0:11662:0:(rw26.c:954:ll_write_end()) page@000000003cc84fea[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.809758:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @000000003c9415da 17ffffc0001001 3:0 ffff9001f7bb8880 548 lru 00000080:00200000:2.0:1761231219.809761:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@00000000aecffa57 292: 1< 0x845fed 0 - - > 2< 1196032 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 0 0 8 485752832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1761231219.809764:0:11662:0:(rw26.c:954:ll_write_end()) end page@000000003cc84fea 00000080:00200000:2.0:1761231219.809765:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 8. 00000080:00200000:2.0:1761231219.809766:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 549 of 0 to 4096 bytes 00000080:00200000:2.0:1761231219.809774:0:11662:0:(rw26.c:916:ll_write_end()) pos 2248704, len 4096, copied 4096 00000080:00200000:2.0:1761231219.809776:0:11662:0:(rw26.c:954:ll_write_end()) page@00000000d81fdf3f[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.809778:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @000000003e8da278 17ffffc0001001 3:0 ffff9001f7bb8770 549 lru 00000080:00200000:2.0:1761231219.809782:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@00000000ea254309 293: 1< 0x845fed 0 - - > 2< 1200128 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 0 0 8 485752832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1761231219.809784:0:11662:0:(rw26.c:954:ll_write_end()) end page@00000000d81fdf3f 00000080:00200000:2.0:1761231219.809784:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 9. 00000080:00200000:2.0:1761231219.809785:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 550 of 0 to 4096 bytes 00000080:00200000:2.0:1761231219.809793:0:11662:0:(rw26.c:916:ll_write_end()) pos 2252800, len 4096, copied 4096 00000080:00200000:2.0:1761231219.809795:0:11662:0:(rw26.c:954:ll_write_end()) page@00000000470cd53c[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.809797:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @00000000c13613f3 17ffffc0001001 3:0 ffff9001f7bb9ed0 550 lru 00000080:00200000:2.0:1761231219.809801:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@0000000038a35a27 294: 1< 0x845fed 0 - - > 2< 1204224 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 0 0 8 485752832 - | + - - + > 5< + - - + | 160 + | 0 - -> 00000080:00200000:2.0:1761231219.809802:0:11662:0:(rw26.c:954:ll_write_end()) end page@00000000470cd53c 00000080:00200000:2.0:1761231219.809803:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 10. 00000080:00200000:2.0:1761231219.809803:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 551 of 0 to 1322 bytes 00000080:00200000:2.0:1761231219.809812:0:11662:0:(rw26.c:916:ll_write_end()) pos 2256896, len 1322, copied 1322 00000080:00200000:2.0:1761231219.809817:0:11662:0:(rw26.c:954:ll_write_end()) page@00000000f6e5eff1[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.809820:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @00000000e5973589 17ffffc0001001 3:0 ffff9001f7bb9320 551 lru 00000080:00200000:2.0:1761231219.809824:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@0000000056c130d9 295: 1< 0x845fed 0 - - > 2< 1208320 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 0 0 8 485752832 - | + - - + > 5< + - - + | 160 + | 0 - -> 00000080:00200000:2.0:1761231219.809847:0:11662:0:(rw26.c:954:ll_write_end()) end page@00000000f6e5eff1 00000080:00200000:2.0:1761231219.809849:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 11. 00000080:00200000:2.0:1761231219.809852:0:11662:0:(vvp_io.c:1207:vvp_io_write_commit()) commit async pages: 11, from 9, to 1322 00000008:00010000:2.0:1761231219.809865:0:11662:0:(osc_cache.c:803:osc_extent_find()) ### extent: 0000000050e63ee7 ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.809903:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 11, dirtied 11 00000080:00200000:2.0:1761231219.809908:0:11662:0:(vvp_io.c:1230:vvp_io_write_commit()) Committed 11 pages 42273 bytes, tot: 42273 00000080:00200000:2.0:1761231219.809914:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 2258218 00000080:00200000:2.0:1761231219.809917:0:11662:0:(vvp_io.c:1395:vvp_io_write_start()) f0.fsxfile: write nob 0, result: 42273 00000020:00010000:2.0:1761231219.809928:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.809930:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.809933:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380ea056 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.809935:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.809951:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.809957:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.809958:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380ea056 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.809962:0:11662:0:(vvp_io.c:538:vvp_io_advance()) advancing 42273 bytes 00000080:00200000:2.0:1761231219.809965:0:11662:0:(file.c:1773:ll_file_io_generic()) Range unlock [541, 551] 00000080:00200000:2.0:1761231219.809969:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.809971:0:11662:0:(file.c:1816:ll_file_io_generic()) f0.fsxfile: 2 io complete with rc: 0, result: 42273, restart: 0 00000080:00200000:2.0:1761231219.809973:0:11662:0:(file.c:1878:ll_file_io_generic()) iot: 2, result: 42273 00000080:00200000:2.0:1761231219.809979:0:11662:0:(file.c:5227:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef),name=f0.fsxfile 00000002:00010000:2.0:1761231219.809984:0:11662:0:(mdc_locks.c:1328:mdc_intent_lock()) (name: ,[0x200000401:0x64f6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1761231219.809988:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231219.809994:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1761231219.810001:0:11662:0:(dcache.c:293:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) 00000080:00010000:2.0:1761231219.810003:0:11662:0:(llite_internal.h:1711:ll_set_lock_data()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) for lock 0x311d04a022a024ba 00000080:00010000:2.0:1761231219.810007:0:11662:0:(dcache.c:193:ll_intent_drop_lock()) releasing lock with cookie 0x311d04a022a024ba from it 0000000056434e97 00010000:00010000:2.0:1761231219.810009:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231219.810016:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1761231219.810032:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231219.810037:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.810044:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.810045:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000dd2129b9, osc lock 0000000016d67cbc, flags 0x400001002 00000008:00010000:2.0:1761231219.810049:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.810050:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000006f8a84e0, osc lock 00000000753cbb65, flags 0x400001002 00010000:00010000:2.0:1761231219.810055:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810063:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810071:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810078:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810089:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810096:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810103:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810115:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.810125:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 2258218 00000020:00010000:2.0:1761231219.810127:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.810129:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.810131:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000dd2129b9 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.810134:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000006f8a84e0 (0000000001a1b4c1/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.810136:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.810138:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000dd2129b9 (00000000c270dd85/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.810141:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000006f8a84e0 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.810147:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.810154:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=0=0x0(2) 00000080:00200000:2.0:1761231219.810158:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231219.810160:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.810166:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.810167:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000dd2129b9, osc lock 00000000753cbb65, flags 0x400001002 00000008:00010000:2.0:1761231219.810170:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.810172:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000006f8a84e0, osc lock 0000000016d67cbc, flags 0x400001002 00010000:00010000:2.0:1761231219.810176:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810181:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810186:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810191:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810197:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810200:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810204:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810207:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.810212:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 2258218 00000020:00010000:2.0:1761231219.810213:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.810214:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.810215:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000dd2129b9 (0000000001a1b4c1/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.810216:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000006f8a84e0 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.810218:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.810219:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000dd2129b9 (00000000c270dd85/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.810221:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000006f8a84e0 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.810224:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.810638:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=4290034=0x4175f2(0) 00000080:00200008:2.0:1761231219.810646:0:11662:0:(file.c:2180:ll_file_write_iter()) file f0.fsxfile:[0x200000401:0x64f6:0x0], ppos: 4290034, count: 15750 00000080:00200000:2.0:1761231219.810649:0:11662:0:(file.c:1671:ll_file_io_generic()) f0.fsxfile: write ppos: 4290034, count: 15750 00000080:00200000:2.0:1761231219.810651:0:11662:0:(file.c:1543:ll_io_set_mirror()) f0.fsxfile: desiginated mirror: 0 00000020:00200000:2.0:1761231219.810655:0:11662:0:(cl_io.c:216:cl_io_rw_init()) header@000000008978f1bb[0x4, 474, [0x200000401:0x64f6:0x0] hash] 00000020:00200000:2.0:1761231219.810656:0:11662:0:(cl_io.c:216:cl_io_rw_init()) io range: 2 [4290034, 4305784) 0 0 00000080:00200000:2.0:1761231219.810658:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1761231219.810663:0:11662:0:(file.c:1736:ll_file_io_generic()) Range lock [1047, 1051] 00020000:00200000:2.0:1761231219.810665:0:11662:0:(lov_io.c:996:lov_io_rw_iter_init()) stripe: 4 chunk: [4290034, 4305784) 4305784, 15750 00020000:00200000:2.0:1761231219.810667:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.810673:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [1047, 1051] 00000008:00010000:2.0:1761231219.810680:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.810683:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000380ea056, osc lock 0000000016d67cbc, flags 0x0 00010000:00010000:2.0:1761231219.810688:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810696:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (2191360 2211839) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810703:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810707:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810711:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.810715:0:11662:0:(vvp_io.c:1318:vvp_io_write_start()) f0.fsxfile: write [4290034, 4305784) 00000080:00200000:2.0:1761231219.810719:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 1047 of 1522 to 2574 bytes 00000080:00200000:2.0:1761231219.810739:0:11662:0:(rw26.c:916:ll_write_end()) pos 4290034, len 2574, copied 2574 00000080:00200000:2.0:1761231219.810742:0:11662:0:(rw26.c:954:ll_write_end()) page@00000000b5358d0f[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.810744:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @000000006cac8ed5 17ffffc0001001 3:0 ffff9001f7bb9100 1047 lru 00000080:00200000:2.0:1761231219.810749:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@00000000545ce730 535: 1< 0x845fed 0 - - > 2< 2191360 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 1 0 8 484048896 - | - - + - > 5< - - + - | 0 - | 11 - -> 00000080:00200000:2.0:1761231219.810750:0:11662:0:(rw26.c:954:ll_write_end()) end page@00000000b5358d0f 00000080:00200000:2.0:1761231219.810751:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 1. 00000080:00200000:2.0:1761231219.810752:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 1048 of 0 to 4096 bytes 00000080:00200000:2.0:1761231219.810759:0:11662:0:(rw26.c:916:ll_write_end()) pos 4292608, len 4096, copied 4096 00000080:00200000:2.0:1761231219.810761:0:11662:0:(rw26.c:954:ll_write_end()) page@000000001c759052[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.810763:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @00000000ff1bee7b 17ffffc0001001 3:0 ffff9001f7bb8550 1048 lru 00000080:00200000:2.0:1761231219.810768:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@00000000f71961f5 536: 1< 0x845fed 0 - - > 2< 2195456 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 1 0 8 484048896 - | - - + - > 5< - - + - | 0 - | 11 - -> 00000080:00200000:2.0:1761231219.810770:0:11662:0:(rw26.c:954:ll_write_end()) end page@000000001c759052 00000080:00200000:2.0:1761231219.810771:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 2. 00000080:00200000:2.0:1761231219.810773:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 1049 of 0 to 4096 bytes 00000080:00200000:2.0:1761231219.810785:0:11662:0:(rw26.c:916:ll_write_end()) pos 4296704, len 4096, copied 4096 00000080:00200000:2.0:1761231219.810789:0:11662:0:(rw26.c:954:ll_write_end()) page@0000000073846f63[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.810792:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @00000000b90d34ed 17ffffc0001001 3:0 ffff9001f7bb8220 1049 lru 00000080:00200000:2.0:1761231219.810796:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@0000000035e65f81 537: 1< 0x845fed 0 - - > 2< 2199552 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 1 0 8 484048896 - | - - + - > 5< - - + - | 0 - | 11 - -> 00000080:00200000:2.0:1761231219.810798:0:11662:0:(rw26.c:954:ll_write_end()) end page@0000000073846f63 00000080:00200000:2.0:1761231219.810799:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 3. 00000080:00200000:2.0:1761231219.810800:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 1050 of 0 to 4096 bytes 00000080:00200000:2.0:1761231219.810818:0:11662:0:(rw26.c:916:ll_write_end()) pos 4300800, len 4096, copied 4096 00000080:00200000:2.0:1761231219.810822:0:11662:0:(rw26.c:954:ll_write_end()) page@000000003f79a144[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.810825:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @00000000c44b4a7b 17ffffc0001021 2:0 ffff9001f7bb8330 1050 lru 00000080:00200000:2.0:1761231219.810850:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@00000000feddd1dd 538: 1< 0x845fed 0 - - > 2< 2203648 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 1 0 8 484048896 - | - - + - > 5< - - + - | 0 - | 11 - -> 00000080:00200000:2.0:1761231219.810852:0:11662:0:(rw26.c:954:ll_write_end()) end page@000000003f79a144 00000080:00200000:2.0:1761231219.810852:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 4. 00000080:00200000:2.0:1761231219.810854:0:11662:0:(rw26.c:730:ll_write_begin()) Writing 1051 of 0 to 888 bytes 00000080:00200000:2.0:1761231219.810872:0:11662:0:(rw26.c:916:ll_write_end()) pos 4304896, len 888, copied 888 00000080:00200000:2.0:1761231219.810875:0:11662:0:(rw26.c:954:ll_write_end()) page@0000000043cc25d1[3 00000000709ff552 2 1 000000005c832229] 00000080:00200000:2.0:1761231219.810878:0:11662:0:(rw26.c:954:ll_write_end()) vmpage @0000000083d4851d 17ffffc0001001 3:0 ffff9001f7bb8cc0 1051 lru 00000080:00200000:2.0:1761231219.810883:0:11662:0:(rw26.c:954:ll_write_end()) osc-page@00000000150aed81 539: 1< 0x845fed 0 - - > 2< 2207744 0 0 0x0 0x0 | 0000000000000000 00000000f02334a0 00000000de8ee77c > 3< 0 0 0 > 4< 1 0 8 484048896 - | - - + - > 5< - - + - | 0 - | 11 - -> 00000080:00200000:2.0:1761231219.810886:0:11662:0:(rw26.c:954:ll_write_end()) end page@0000000043cc25d1 00000080:00200000:2.0:1761231219.810887:0:11662:0:(rw26.c:954:ll_write_end()) queued page: 5. 00000080:00200000:2.0:1761231219.810889:0:11662:0:(vvp_io.c:1207:vvp_io_write_commit()) commit async pages: 5, from 1522, to 888 00000008:00010000:2.0:1761231219.810898:0:11662:0:(osc_cache.c:803:osc_extent_find()) ### extent: 000000002f66cef9 ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.810911:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 5, dirtied 5 00000080:00200000:2.0:1761231219.810913:0:11662:0:(vvp_io.c:1230:vvp_io_write_commit()) Committed 5 pages 15750 bytes, tot: 15750 00000080:00200000:2.0:1761231219.810916:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4305784 00000080:00200000:2.0:1761231219.810918:0:11662:0:(vvp_io.c:1395:vvp_io_write_start()) f0.fsxfile: write nob 0, result: 15750 00000020:00010000:2.0:1761231219.810923:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.810925:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.810927:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380ea056 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.810930:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.810937:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.810945:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.810948:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380ea056 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.810952:0:11662:0:(vvp_io.c:538:vvp_io_advance()) advancing 15750 bytes 00000080:00200000:2.0:1761231219.810956:0:11662:0:(file.c:1773:ll_file_io_generic()) Range unlock [1047, 1051] 00000080:00200000:2.0:1761231219.810959:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.810961:0:11662:0:(file.c:1816:ll_file_io_generic()) f0.fsxfile: 2 io complete with rc: 0, result: 15750, restart: 0 00000080:00200000:2.0:1761231219.810964:0:11662:0:(file.c:1878:ll_file_io_generic()) iot: 2, result: 15750 00000080:00200000:2.0:1761231219.810970:0:11662:0:(file.c:5227:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef),name=f0.fsxfile 00000002:00010000:2.0:1761231219.810978:0:11662:0:(mdc_locks.c:1328:mdc_intent_lock()) (name: ,[0x200000401:0x64f6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1761231219.810984:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231219.810991:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1761231219.810997:0:11662:0:(dcache.c:293:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) 00000080:00010000:2.0:1761231219.810999:0:11662:0:(llite_internal.h:1711:ll_set_lock_data()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) for lock 0x311d04a022a024ba 00000080:00010000:2.0:1761231219.811002:0:11662:0:(dcache.c:193:ll_intent_drop_lock()) releasing lock with cookie 0x311d04a022a024ba from it 0000000056434e97 00010000:00010000:2.0:1761231219.811004:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231219.811010:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1761231219.811020:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231219.811023:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.811029:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.811030:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000dd2129b9, osc lock 0000000016d67cbc, flags 0x400001002 00000008:00010000:2.0:1761231219.811034:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.811035:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000006f8a84e0, osc lock 00000000753cbb65, flags 0x400001002 00010000:00010000:2.0:1761231219.811040:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811047:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811053:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811060:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811066:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811069:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811074:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811077:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.811082:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4305784 00000020:00010000:2.0:1761231219.811084:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.811085:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.811086:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000dd2129b9 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.811087:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000006f8a84e0 (0000000001a1b4c1/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.811088:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.811089:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000dd2129b9 (00000000c270dd85/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.811091:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000006f8a84e0 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.811094:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.811099:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=0=0x0(2) 00000080:00200000:2.0:1761231219.811101:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231219.811103:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.811106:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.811107:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000dd2129b9, osc lock 00000000753cbb65, flags 0x400001002 00000008:00010000:2.0:1761231219.811109:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.811109:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000006f8a84e0, osc lock 0000000016d67cbc, flags 0x400001002 00010000:00010000:2.0:1761231219.811112:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811116:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811120:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811123:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811128:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811131:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811135:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811138:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.811144:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4305784 00000020:00010000:2.0:1761231219.811145:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.811146:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.811147:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000dd2129b9 (0000000001a1b4c1/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.811149:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000006f8a84e0 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.811150:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.811151:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000dd2129b9 (00000000c270dd85/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.811152:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000006f8a84e0 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.811155:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00a00000:2.0:1761231219.811191:0:11662:0:(llite_mmap.c:603:ll_file_mmap()) VFS_Op: fid=[0x200000401:0x64f6:0x0] vma=0000000023b3f154 start=0x7fbc0645a000 end=0x7fbc0645e000 vm_flags=0xfb 00000080:00200000:2.0:1761231219.811198:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231219.811201:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.811206:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.811207:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000dd2129b9, osc lock 0000000016d67cbc, flags 0x400001002 00000008:00010000:2.0:1761231219.811210:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.811211:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000006f8a84e0, osc lock 00000000753cbb65, flags 0x400001002 00010000:00010000:2.0:1761231219.811215:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811222:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811230:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811236:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811245:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811253:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811259:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811265:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.811275:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4305784 00000020:00010000:2.0:1761231219.811277:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.811279:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.811281:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000dd2129b9 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.811283:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000006f8a84e0 (0000000001a1b4c1/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.811285:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.811287:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000dd2129b9 (00000000c270dd85/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.811290:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000006f8a84e0 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.811296:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.811328:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 906 00000080:00200000:2.0:1761231219.811333:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.811338:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.811343:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [906, 906] 00000008:00010000:2.0:1761231219.811348:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.811350:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000380ea056, osc lock 00000000753cbb65, flags 0x0 00010000:00010000:2.0:1761231219.811356:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811364:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1613824 1617919) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811371:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811377:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.811383:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1761231219.811446:0:2230:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000008:00010000:2.0:1761231219.811454:0:2230:0:(osc_cache.c:1877:try_to_add_extent_for_io()) ### no dlm lock: extent: 0000000081490aef 00010000:00010000:2.0:1761231219.811467:0:2230:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1613824 1617919) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00100000:2.0:1761231219.811483:0:2230:0:(osc_request.c:1944:osc_brw_prep_request()) brw rpc 00000000f38f27df - object 0x0:23052 offset 1613824<>1617920 00000100:00100000:2.0:1761231219.811500:0:2230:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606980096:192.168.201.104@tcp:3:fsx.0 00000100:00100000:2.0:1761231219.812900:0:2230:0:(client.c:2868:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9001d2597000: committing for last_committed 4295030203 gen 4 00000008:00010000:2.0:1761231219.812959:0:2230:0:(osc_cache.c:843:osc_extent_finish()) ### no dlm lock: extent: 0000000081490aef 00000008:00010000:2.0:1761231219.812981:0:2230:0:(osc_cache.c:503:__osc_extent_remove()) ### no dlm lock: extent: 0000000081490aef 00000100:00100000:2.0:1761231219.812989:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606980096:192.168.201.104@tcp:3:fsx.0 00000020:00010000:2.0:1761231219.814334:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.814337:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.814339:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000005c67ef96 (0000000001a1b4c1/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.814342:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.814349:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.814355:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.814356:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000005c67ef96 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.814363:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.814372:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.814376:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.814380:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [907, 907] 00000008:00010000:2.0:1761231219.814385:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.814386:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000d204fb08, osc lock 0000000016d67cbc, flags 0x0 00010000:00010000:2.0:1761231219.814391:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.814397:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1617920 1622015) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.814413:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.814420:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.814428:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231219.814441:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 00000000b865ce3f ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.814450:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231219.814455:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.814456:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.814458:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d204fb08 (0000000001a1b4c1/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.814460:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.814463:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.814468:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.814469:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d204fb08 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.814473:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.814490:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 908 00000080:00200000:2.0:1761231219.814493:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.814497:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.814500:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [908, 908] 00000008:00010000:2.0:1761231219.814504:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.814505:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000d204fb08, osc lock 0000000016d67cbc, flags 0x0 00010000:00010000:2.0:1761231219.814509:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.814517:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1622016 1626111) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.814522:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.814525:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.814528:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1761231219.814564:0:2230:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000008:00010000:2.0:1761231219.814572:0:2230:0:(osc_cache.c:1877:try_to_add_extent_for_io()) ### no dlm lock: extent: 0000000081490aef 00010000:00010000:2.0:1761231219.814583:0:2230:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1622016 1626111) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00100000:2.0:1761231219.814601:0:2230:0:(osc_request.c:1944:osc_brw_prep_request()) brw rpc 00000000f38f27df - object 0x0:23052 offset 1622016<>1626112 00000100:00100000:2.0:1761231219.814635:0:2230:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606980352:192.168.201.104@tcp:3:fsx.0 00000008:00010000:2.0:1761231219.815467:0:2230:0:(osc_cache.c:843:osc_extent_finish()) ### no dlm lock: extent: 0000000081490aef 00000008:00010000:2.0:1761231219.815479:0:2230:0:(osc_cache.c:503:__osc_extent_remove()) ### no dlm lock: extent: 0000000081490aef 00000100:00100000:2.0:1761231219.815484:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606980352:192.168.201.104@tcp:3:fsx.0 00000020:00010000:2.0:1761231219.816557:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.816561:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.816564:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000001e7ce90f (0000000001a1b4c1/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.816568:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.816576:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.816585:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.816587:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000001e7ce90f (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.816596:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.816618:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.816624:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.816629:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [909, 909] 00000008:00010000:2.0:1761231219.816635:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.816638:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000a29cb894, osc lock 0000000094032a5c, flags 0x0 00010000:00010000:2.0:1761231219.816643:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.816651:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1626112 1630207) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.816658:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.816664:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.816670:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231219.816691:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 00000000b865ce3f ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.816704:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231219.816710:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.816713:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.816714:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000a29cb894 (0000000001a1b4c1/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.816717:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.816723:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.816731:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.816733:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000a29cb894 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.816740:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.816752:0:11662:0:(file.c:4709:ll_fsync()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), start 3710976, end 3727359, datasync 1 00000080:00200000:2.0:1761231219.816759:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.816763:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1761231219.816770:0:11662:0:(osc_cache.c:1877:try_to_add_extent_for_io()) ### extent: 00000000b865ce3f ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231219.816778:0:11662:0:(osc_cache.c:1084:osc_extent_make_ready()) ### extent: 00000000b865ce3f ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.816810:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1613824 1617919) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00100000:2.0:1761231219.816879:0:11662:0:(osc_request.c:1944:osc_brw_prep_request()) brw rpc 00000000f38f27df - object 0x0:23052 offset 1613824<>1630208 00000100:00100000:2.0:1761231219.816908:0:2230:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761231219.816918:0:2230:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606980544:192.168.201.104@tcp:4:fsx.0 00000008:00010000:2.0:1761231219.818015:0:2230:0:(osc_cache.c:843:osc_extent_finish()) ### extent: 00000000b865ce3f ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231219.818042:0:2230:0:(osc_cache.c:503:__osc_extent_remove()) ### extent: 00000000b865ce3f ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1761231219.818049:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606980544:192.168.201.104@tcp:4:fsx.0 00000080:00200000:2.0:1761231219.827993:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.828052:0:11662:0:(file.c:5227:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef),name=f0.fsxfile 00000002:00010000:2.0:1761231219.828069:0:11662:0:(mdc_locks.c:1328:mdc_intent_lock()) (name: ,[0x200000401:0x64f6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1761231219.828084:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231219.828096:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1761231219.828105:0:11662:0:(dcache.c:293:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) 00000080:00010000:2.0:1761231219.828108:0:11662:0:(llite_internal.h:1711:ll_set_lock_data()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) for lock 0x311d04a022a024ba 00000080:00010000:2.0:1761231219.828114:0:11662:0:(dcache.c:193:ll_intent_drop_lock()) releasing lock with cookie 0x311d04a022a024ba from it 0000000056434e97 00010000:00010000:2.0:1761231219.828116:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231219.828128:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000982b00af/0x311d04a022a024ba lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093076d expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1761231219.828146:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231219.828154:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.828169:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.828172:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000dd2129b9, osc lock 0000000094032a5c, flags 0x400001002 00000008:00010000:2.0:1761231219.828177:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.828178:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000006f8a84e0, osc lock 0000000016a6be7a, flags 0x400001002 00010000:00010000:2.0:1761231219.828190:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828198:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828208:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828214:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828227:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828243:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828259:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828266:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.828283:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4305784 00000020:00010000:2.0:1761231219.828286:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.828289:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.828292:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000dd2129b9 (0000000001a1b4c1/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.828296:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000006f8a84e0 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.828299:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.828301:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000dd2129b9 (00000000c270dd85/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.828304:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000006f8a84e0 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.828310:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.828319:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=0=0x0(2) 00000080:00200000:2.0:1761231219.828322:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231219.828324:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.828328:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.828329:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000dd2129b9, osc lock 0000000016a6be7a, flags 0x400001002 00000008:00010000:2.0:1761231219.828331:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.828332:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000006f8a84e0, osc lock 0000000094032a5c, flags 0x400001002 00010000:00010000:2.0:1761231219.828336:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828341:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828346:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828350:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828355:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828359:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828363:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828366:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.828370:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4305784 00000020:00010000:2.0:1761231219.828372:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.828373:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.828374:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000dd2129b9 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.828375:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000006f8a84e0 (0000000001a1b4c1/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.828376:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.828377:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000dd2129b9 (00000000c270dd85/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.828379:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000006f8a84e0 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.828382:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.828436:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=3996250=0x3cfa5a(0) 00000080:00200008:2.0:1761231219.828443:0:11662:0:(file.c:2028:ll_file_read_iter()) file f0.fsxfile:[0x200000401:0x64f6:0x0], ppos: 3996250, count: 64648 00000080:00200000:2.0:1761231219.828468:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 975 00000080:00200000:2.0:1761231219.828474:0:11662:0:(file.c:1671:ll_file_io_generic()) f0.fsxfile: read ppos: 3996250, count: 64648 00000080:00200000:2.0:1761231219.828477:0:11662:0:(file.c:1543:ll_io_set_mirror()) f0.fsxfile: desiginated mirror: 0 00000020:00200000:2.0:1761231219.828480:0:11662:0:(cl_io.c:216:cl_io_rw_init()) header@000000008978f1bb[0x4, 483, [0x200000401:0x64f6:0x0] hash] 00000020:00200000:2.0:1761231219.828482:0:11662:0:(cl_io.c:216:cl_io_rw_init()) io range: 1 [3996250, 4060898) 0 0 00000080:00200000:2.0:1761231219.828484:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.828494:0:11662:0:(lov_io.c:996:lov_io_rw_iter_init()) stripe: 3 chunk: [3996250, 4060898) 4060898, 64648 00020000:00200000:2.0:1761231219.828498:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.828509:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [975, 991] 00000008:00010000:2.0:1761231219.828518:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.828520:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000a29cb894, osc lock 0000000094032a5c, flags 0x0 00010000:00010000:2.0:1761231219.828525:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828533:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1896448 1966079) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828541:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828548:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828554:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.828562:0:11662:0:(vvp_io.c:829:vvp_io_read_start()) f0.fsxfile: read [3996250, 4060898) 00000080:00200000:2.0:1761231219.828572:0:11662:0:(vvp_io.c:886:vvp_io_read_start()) count:64648 iocb pos:3996250 00000080:00200000:2.0:1761231219.828575:0:11662:0:(rw.c:1924:ll_readpage()) pgno:975, beyond read end_index:992 00010000:00010000:2.0:1761231219.828603:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828611:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1900544 1904639) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828619:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/2,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828625:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/2,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.828630:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/2,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.829002:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/2,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.829011:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.829023:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.829030:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (2097152 2101247) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.829041:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.829047:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.829053:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.829282:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.829290:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231219.831935:0:2230:0:(osc_cache.c:843:osc_extent_finish()) ### no dlm lock: extent: 00000000cc0bb326 00000008:00010000:2.0:1761231219.831955:0:2230:0:(osc_cache.c:503:__osc_extent_remove()) ### no dlm lock: extent: 00000000cc0bb326 00000100:00100000:2.0:1761231219.831966:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606980928:192.168.201.104@tcp:3:fsx.0 00000008:00010000:2.0:1761231219.832059:0:2230:0:(osc_cache.c:843:osc_extent_finish()) ### no dlm lock: extent: 0000000081490aef 00000008:00010000:2.0:1761231219.832086:0:2230:0:(osc_cache.c:503:__osc_extent_remove()) ### no dlm lock: extent: 0000000081490aef 00000100:00100000:2.0:1761231219.832093:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606980864:192.168.201.104@tcp:3:fsx.0 00000020:00010000:2.0:1761231219.837167:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.837173:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.837177:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000002f694ed5 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.837184:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.837197:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.837207:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000065f22fda (0000000001a1b4c1/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.837210:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.837214:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.837218:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.837219:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000002f694ed5 (00000000c270dd85/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.837222:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000065f22fda (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.837233:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.837255:0:11662:0:(file.c:5227:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef),name=f0.fsxfile 00000002:00010000:2.0:1761231219.837268:0:11662:0:(mdc_locks.c:1328:mdc_intent_lock()) (name: ,[0x200000401:0x64f6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1761231219.837276:0:11662:0:(ldlm_lock.c:1511:ldlm_lock_match_with_skip()) ### not matched ns 0000000072b82a68 type 13 mode 30 res 8589935617/25846 (0 0) 00010000:00010000:2.0:1761231219.837303:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000a4d6f64f/0x311d04a022a024c1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1761231219.837308:0:11662:0:(ldlm_request.c:1026:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000a4d6f64f/0x311d04a022a024c1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1761231219.837313:0:11662:0:(ldlm_request.c:1114:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 00000000a4d6f64f/0x311d04a022a024c1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1761231219.837326:0:11662:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job fsx:0995b904-d672-4cf9-8bdd-5554fffeedd5:11662:1846780606981248:192.168.201.104@tcp:101:fsx.0 00000100:00100000:2.0:1761231219.837370:0:11662:0:(client.c:2500:ptlrpc_set_wait()) set 0000000069b656d9 going to sleep for 43 seconds 00000020:00010000:2.0:1761231219.841881:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000e4987176 (00000000e3767fbf/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.841886:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000e4987176 (00000000e3767fbf/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.841889:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000053d908cd (0000000022f7d55b/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.841893:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.841900:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.841909:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d9e5c498 (00000000a29972cd/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.841911:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.841915:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.841918:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000e4987176 (00000000e3767fbf/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.841920:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000053d908cd (00000000e3767fbf/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.841922:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d9e5c498 (00000000e3767fbf/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.841958:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00a00000:2.0:1761231219.841989:0:11662:0:(llite_mmap.c:603:ll_file_mmap()) VFS_Op: fid=[0x200000401:0x64f6:0x0] vma=0000000023b3f154 start=0x7fbc0644a000 end=0x7fbc06455000 vm_flags=0xfb 00000080:00200000:2.0:1761231219.841997:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231219.842001:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.842007:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.842008:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 0000000044cac728, osc lock 0000000016a6be7a, flags 0x400001002 00000008:00010000:2.0:1761231219.842011:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.842012:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000f15c7de1, osc lock 00000000083d38fd, flags 0x400001002 00010000:00010000:2.0:1761231219.842018:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.842026:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.842034:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.842040:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.842049:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.842056:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.842062:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.842068:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.842081:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 1804169 00000020:00010000:2.0:1761231219.842083:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.842085:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.842088:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000044cac728 (0000000001a1b4c1/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.842091:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000f15c7de1 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.842093:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.842095:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000044cac728 (00000000c270dd85/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.842098:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000f15c7de1 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.842103:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.842154:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 430 00000080:00200000:2.0:1761231219.842160:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.842166:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.842178:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [430, 430] 00000008:00010000:2.0:1761231219.842185:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.842187:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000004aca64ca, osc lock 00000000083d38fd, flags 0x0 00010000:00010000:2.0:1761231219.842193:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.842201:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (712704 716799) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.842208:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.842212:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.842218:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.843482:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.843486:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.843489:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000004aca64ca (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.843491:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.843497:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.843504:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.843505:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000004aca64ca (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.843512:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.843526:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.843530:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.843533:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [430, 430] 00000008:00010000:2.0:1761231219.843538:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.843540:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000004aca64ca, osc lock 00000000083d38fd, flags 0x0 00010000:00010000:2.0:1761231219.843545:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.843550:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (712704 716799) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.843555:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.843559:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.843562:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231219.843582:0:11662:0:(osc_cache.c:803:osc_extent_find()) ### extent: 00000000cc0bb326 ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.843595:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231219.843602:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.843604:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.843605:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000004aca64ca (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.843606:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.843610:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.843614:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.843616:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000004aca64ca (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.843619:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.843639:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 431 00000080:00200000:2.0:1761231219.843642:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.843645:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.843648:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [431, 431] 00000008:00010000:2.0:1761231219.843652:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.843653:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000004aca64ca, osc lock 00000000083d38fd, flags 0x0 00010000:00010000:2.0:1761231219.843656:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.843661:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (716800 720895) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.843670:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.843674:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.843677:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.844697:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.844701:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.844703:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000004aca64ca (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.844706:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.844711:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.844717:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.844718:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000004aca64ca (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.844724:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.844733:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.844737:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.844740:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [431, 431] 00000008:00010000:2.0:1761231219.844745:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.844746:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000004aca64ca, osc lock 00000000083d38fd, flags 0x0 00010000:00010000:2.0:1761231219.844751:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.844756:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (716800 720895) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.844760:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.844764:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.844767:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231219.844779:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 00000000cc0bb326 ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.844787:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231219.844791:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.844793:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.844794:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000004aca64ca (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.844796:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.844799:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.844808:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.844809:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000004aca64ca (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.844813:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.844841:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 432 00000080:00200000:2.0:1761231219.844845:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.844849:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.844854:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [432, 432] 00000008:00010000:2.0:1761231219.844860:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.844862:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000004aca64ca, osc lock 00000000083d38fd, flags 0x0 00010000:00010000:2.0:1761231219.844868:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.844875:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (720896 724991) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.844883:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.844889:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.844894:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.845786:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.845790:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.845792:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000004aca64ca (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.845795:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.845801:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.845807:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.845808:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000004aca64ca (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.845815:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.845844:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.845852:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.845857:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [432, 432] 00000008:00010000:2.0:1761231219.845864:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.845866:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000004aca64ca, osc lock 00000000083d38fd, flags 0x0 00010000:00010000:2.0:1761231219.845872:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.845880:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (720896 724991) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.845888:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.845894:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.845899:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231219.845921:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 00000000cc0bb326 ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.845934:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231219.845941:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.845943:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.845945:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000004aca64ca (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.845948:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.845955:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.845963:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.845965:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000004aca64ca (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.845972:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.845995:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 433 00000080:00200000:2.0:1761231219.845999:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.846004:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.846009:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [433, 433] 00000008:00010000:2.0:1761231219.846014:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.846016:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000004aca64ca, osc lock 00000000083d38fd, flags 0x0 00010000:00010000:2.0:1761231219.846022:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.846029:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (724992 729087) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.846043:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.846047:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.846050:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.846850:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.846857:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.846860:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000004aca64ca (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.846863:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.846879:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.846892:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.846894:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000004aca64ca (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.846902:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.846910:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.846914:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.846917:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [433, 433] 00000008:00010000:2.0:1761231219.846921:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.846923:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000004aca64ca, osc lock 00000000083d38fd, flags 0x0 00010000:00010000:2.0:1761231219.846927:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.846932:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (724992 729087) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.846937:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.846941:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.846944:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231219.846954:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 00000000cc0bb326 ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.846963:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231219.846967:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.846969:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.846970:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000004aca64ca (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.846972:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.846975:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.846986:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.846988:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000004aca64ca (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.846994:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.847013:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 434 00000080:00200000:2.0:1761231219.847018:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.847022:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.847027:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [434, 434] 00000008:00010000:2.0:1761231219.847032:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.847034:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000004aca64ca, osc lock 00000000083d38fd, flags 0x0 00010000:00010000:2.0:1761231219.847039:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.847046:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (729088 733183) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.847052:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.847056:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.847060:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.857276:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0 00000100:00100000:2.0:1761231219.857316:0:11662:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job fsx:0995b904-d672-4cf9-8bdd-5554fffeedd5:11662:1846780606983040:192.168.201.104@tcp:44:fsx.0 00000100:00100000:2.0:1761231219.857358:0:11662:0:(client.c:2500:ptlrpc_set_wait()) set 0000000025f08f4b going to sleep for 43 seconds 00000100:00100000:2.0:1761231219.858143:0:11662:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job fsx:0995b904-d672-4cf9-8bdd-5554fffeedd5:11662:1846780606983040:192.168.201.104@tcp:44:fsx.0 00000080:00200000:2.0:1761231219.858162:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.858170:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:2.0:1761231219.868388:0:2230:0:(client.c:2868:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9001d2597000: committing for last_committed 4295030207 gen 4 00000100:00100000:2.0:1761231219.868464:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606983104:192.168.201.104@tcp:16:fsx.0 00000020:00010000:2.0:1761231219.876631:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.876635:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.876637:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000ab68dc7c (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.876639:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.876645:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.876651:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.876653:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000ab68dc7c (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.876659:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.876667:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.876671:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.876674:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [693, 693] 00000008:00010000:2.0:1761231219.876678:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.876680:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000007f3c642b, osc lock 0000000016a6be7a, flags 0x0 00010000:00010000:2.0:1761231219.876684:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.876690:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1789952 1794047) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.876695:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.876699:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.876702:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231219.876712:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 00000000670e8def ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.876721:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231219.876725:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.876727:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.876728:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000007f3c642b (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.876730:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.876733:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.876737:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.876739:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000007f3c642b (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.876743:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.876763:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 694 00000080:00200000:2.0:1761231219.876766:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.876770:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.876772:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [694, 694] 00000008:00010000:2.0:1761231219.876776:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.876777:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000007f3c642b, osc lock 0000000016a6be7a, flags 0x0 00010000:00010000:2.0:1761231219.876781:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.876786:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1794048 1798143) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.876791:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.876794:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.876797:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.877695:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.877698:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.877700:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000007f3c642b (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.877703:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.877708:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.877718:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.877719:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000007f3c642b (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.877725:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.877734:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.877738:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.877741:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [694, 694] 00000008:00010000:2.0:1761231219.877746:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.877747:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000007f3c642b, osc lock 0000000016a6be7a, flags 0x0 00010000:00010000:2.0:1761231219.877752:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.877757:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1794048 1798143) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.877763:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.877769:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.877775:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231219.877791:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 00000000670e8def ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.877804:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231219.877811:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.877814:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.877816:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000007f3c642b (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.877818:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.877823:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.877849:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.877852:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000007f3c642b (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.877860:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.877882:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 695 00000080:00200000:2.0:1761231219.877886:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.877891:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.877895:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [695, 695] 00000008:00010000:2.0:1761231219.877900:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.877902:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000007f3c642b, osc lock 0000000016a6be7a, flags 0x0 00010000:00010000:2.0:1761231219.877908:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.877916:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1798144 1802239) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.877923:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.877927:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.877935:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1761231219.877970:0:2230:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000008:00010000:2.0:1761231219.877976:0:2230:0:(osc_cache.c:1877:try_to_add_extent_for_io()) ### no dlm lock: extent: 00000000434b2013 00010000:00010000:2.0:1761231219.877985:0:2230:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1798144 1802239) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00100000:2.0:1761231219.877999:0:2230:0:(osc_request.c:1944:osc_brw_prep_request()) brw rpc 0000000017386078 - object 0x0:23051 offset 1798144<>1802240 00000100:00100000:2.0:1761231219.878015:0:2230:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606984064:192.168.201.104@tcp:3:fsx.0 00000020:00010000:2.0:1761231219.878843:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.878847:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.878849:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000007f3c642b (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.878852:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.878858:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.878865:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.878866:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000007f3c642b (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.878872:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.878881:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.878884:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.878887:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [695, 695] 00000008:00010000:2.0:1761231219.878892:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.878893:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000007f3c642b, osc lock 0000000016a6be7a, flags 0x0 00010000:00010000:2.0:1761231219.878897:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.878902:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1798144 1802239) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.878907:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.878910:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.878913:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231219.878924:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 00000000670e8def ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.878932:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231219.878937:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.878938:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.878940:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000007f3c642b (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.878941:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.878945:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.878950:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000033a535f5 (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.878952:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000007f3c642b (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.878955:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.878971:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 696 00000080:00200000:2.0:1761231219.878974:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.878977:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.878980:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [696, 696] 00000008:00010000:2.0:1761231219.878984:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.878985:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000007f3c642b, osc lock 0000000016a6be7a, flags 0x0 00010000:00010000:2.0:1761231219.878989:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.878994:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1802240 1806335) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.878998:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.879002:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.879005:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000001:00010000:2.0:1761231219.879016:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.879019:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.879020:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000a3976d55, osc lock 000000006ed9356c, flags 0x400001002 00000008:00010000:2.0:1761231219.879024:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.879025:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000001d55a2d4, osc lock 00000000bca714f2, flags 0x400001002 00010000:00010000:2.0:1761231219.879028:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.879032:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.879036:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.879039:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.879044:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.879047:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.879050:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.879054:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.879060:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 2853080 00000020:00010000:2.0:1761231219.879062:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.879063:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.879065:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000a3976d55 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.879066:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000001d55a2d4 (0000000001a1b4c1/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.879068:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000015ee36bf (00000000c270dd85/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.879069:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000a3976d55 (00000000c270dd85/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.879070:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000001d55a2d4 (00000000c270dd85/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.881912:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0 00000100:00100000:2.0:1761231219.881940:0:11662:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job fsx:0995b904-d672-4cf9-8bdd-5554fffeedd5:11662:1846780606984320:192.168.201.104@tcp:44:fsx.0 00000100:00100000:2.0:1761231219.881967:0:11662:0:(client.c:2500:ptlrpc_set_wait()) set 000000005eb70e32 going to sleep for 43 seconds 00000100:00100000:2.0:1761231219.882589:0:11662:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job fsx:0995b904-d672-4cf9-8bdd-5554fffeedd5:11662:1846780606984320:192.168.201.104@tcp:44:fsx.0 00000080:00200000:2.0:1761231219.882600:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.882604:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:2.0:1761231219.892516:0:2230:0:(client.c:2868:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9001d2597000: committing for last_committed 4295030256 gen 4 00000100:00100000:2.0:1761231219.892566:0:2230:0:(client.c:2894:ptlrpc_free_committed()) @@@ stopping search req@0000000099514377 x1846780606984256/t4295030257(4295030257) o4->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:6/4 lens 504/448 e 0 to 0 dl 1761231263 ref 1 fl Complete:RQU/4/0 rc 0/0 job:'fsx.0' 00000100:00100000:2.0:1761231219.892600:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606984384:192.168.201.104@tcp:16:fsx.0 00000080:00200000:2.0:1761231219.940657:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0 00000100:00100000:2.0:1761231219.940693:0:11662:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job fsx:0995b904-d672-4cf9-8bdd-5554fffeedd5:11662:1846780606986368:192.168.201.104@tcp:44:fsx.0 00000100:00100000:2.0:1761231219.940725:0:11662:0:(client.c:2500:ptlrpc_set_wait()) set 000000005eb70e32 going to sleep for 43 seconds 00000020:00010000:2.0:1761231219.962955:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 0000000014b87993 (0000000001a1b4c1/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.962960:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 0000000014b87993 (0000000001a1b4c1/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.962964:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fd674dca (00000000495e9d75/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.962970:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.962983:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.962996:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000913d6330 (000000009f2cfa08/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231219.962999:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.963006:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231219.963013:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 0000000014b87993 (0000000001a1b4c1/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.963015:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fd674dca (0000000001a1b4c1/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.963019:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000913d6330 (0000000001a1b4c1/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.963052:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.963087:0:11662:0:(file.c:5227:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef),name=f0.fsxfile 00000002:00010000:2.0:1761231219.963100:0:11662:0:(mdc_locks.c:1328:mdc_intent_lock()) (name: ,[0x200000401:0x64f6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1761231219.963109:0:11662:0:(ldlm_lock.c:1511:ldlm_lock_match_with_skip()) ### not matched ns 0000000072b82a68 type 13 mode 30 res 8589935617/25846 (0 0) 00010000:00010000:2.0:1761231219.963133:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1761231219.963140:0:11662:0:(ldlm_request.c:1026:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1761231219.963149:0:11662:0:(ldlm_request.c:1114:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1761231219.963161:0:11662:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b74af89 pname:cluuid:pid:xid:nid:opc:job fsx:0995b904-d672-4cf9-8bdd-5554fffeedd5:11662:1846780606987136:192.168.201.104@tcp:101:fsx.0 00000100:00100000:2.0:1761231219.963204:0:11662:0:(client.c:2500:ptlrpc_set_wait()) set 0000000089d178b2 going to sleep for 43 seconds 00000100:00100000:2.0:1761231219.964065:0:11662:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b74af89 pname:cluuid:pid:xid:nid:opc:job fsx:0995b904-d672-4cf9-8bdd-5554fffeedd5:11662:1846780606987136:192.168.201.104@tcp:101:fsx.0 00010000:00010000:2.0:1761231219.964076:0:11662:0:(ldlm_request.c:714:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1761231219.964086:0:11662:0:(ldlm_request.c:729:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1761231219.964090:0:11662:0:(ldlm_request.c:742:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1761231219.964096:0:11662:0:(ldlm_lock.c:1064:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1761231219.964101:0:11662:0:(ldlm_request.c:796:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1761231219.964107:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1761231219.964111:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1761231219.964114:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1761231219.964120:0:11662:0:(mdc_locks.c:743:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@000000009b74af89 x1846780606987136/t0(0) o101->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 576/720 e 0 to 0 dl 1761231263 ref 1 fl Complete:RQU/0/0 rc 0/0 job:'fsx.0' 00000002:00010000:2.0:1761231219.964125:0:11662:0:(mdc_locks.c:847:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 80 ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 0 00000002:00010000:2.0:1761231219.964133:0:11662:0:(mdc_locks.c:1179:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231219.964137:0:11662:0:(ldlm_lock.c:1511:ldlm_lock_match_with_skip()) ### not matched ns 0000000072b82a68 type 13 mode 4 res 8589935617/25846 (0 0) 00000080:00200000:2.0:1761231219.964167:0:11662:0:(file.c:5870:ll_layout_conf()) [0x200000401:0x64f6:0x0]: layout version change: 0 -> 0 00000080:00010000:2.0:1761231219.964171:0:11662:0:(llite_internal.h:1711:ll_set_lock_data()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) for lock 0x311d04a022a024f2 00000080:00010000:2.0:1761231219.964175:0:11662:0:(dcache.c:293:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) 00000080:00010000:2.0:1761231219.964176:0:11662:0:(dcache.c:193:ll_intent_drop_lock()) releasing lock with cookie 0x311d04a022a024f2 from it 0000000056434e97 00010000:00010000:2.0:1761231219.964177:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231219.964181:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1761231219.964195:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231219.964200:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.964211:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.964213:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000f006e797, osc lock 00000000bca714f2, flags 0x400001002 00000008:00010000:2.0:1761231219.964218:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.964219:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c3d1a1c5, osc lock 000000006ed9356c, flags 0x400001002 00010000:00010000:2.0:1761231219.964228:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964236:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964245:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964251:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964258:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964262:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964266:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964269:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.964280:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 2145254 00000020:00010000:2.0:1761231219.964281:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.964283:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.964284:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000f006e797 (00000000c270dd85/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.964286:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c3d1a1c5 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.964287:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.964288:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000f006e797 (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.964291:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c3d1a1c5 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.964295:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.964302:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=0=0x0(2) 00000080:00200000:2.0:1761231219.964306:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231219.964310:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231219.964315:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.964316:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000f006e797, osc lock 000000006ed9356c, flags 0x400001002 00000008:00010000:2.0:1761231219.964320:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231219.964321:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c3d1a1c5, osc lock 00000000bca714f2, flags 0x400001002 00010000:00010000:2.0:1761231219.964326:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964332:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964338:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964344:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964351:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964357:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964363:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.964369:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231219.964378:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 2145254 00000020:00010000:2.0:1761231219.964380:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231219.964382:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.964383:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000f006e797 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.964385:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c3d1a1c5 (00000000c270dd85/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231219.964387:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.964389:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000f006e797 (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231219.964391:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c3d1a1c5 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231219.964396:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231219.965167:0:11662:0:(llite_lib.c:2191:ll_setattr_raw()) lustre: setattr inode [0x200000401:0x64f6:0x0](0000000047dd89ef) from 2145254 to 5406261, valid 2068, hsm_import 0 00010000:00010000:2.0:1761231219.965188:0:11662:0:(ldlm_request.c:1232:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231219.965202:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000008bf8410e/0x311d04a0229f468f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x64f6:0x0].0x0 bits 0x9/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb90906fe7 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231219.965230:0:11662:0:(ldlm_request.c:1279:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231219.965235:0:11662:0:(ldlm_request.c:1325:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1761231219.965237:0:11662:0:(ldlm_lock.c:216:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000000c26b3b5/0x311d04a022a024f2 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc2e856eb90930815 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000020:00100000:2.0:1761231219.965252:0:11662:0:(genops.c:2339:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9001d2597000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1761231219.965259:0:11662:0:(client.c:742:ptlrpc_reassign_next_xid()) @@@ reassign xid req@000000009b74af89 x1846780606987264/t0(0) o36->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761231219.965273:0:11662:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b74af89 pname:cluuid:pid:xid:nid:opc:job fsx:0995b904-d672-4cf9-8bdd-5554fffeedd5:11662:1846780606987264:192.168.201.104@tcp:36:fsx.0 00000100:00100000:2.0:1761231219.965328:0:11662:0:(client.c:2500:ptlrpc_set_wait()) set 0000000089d178b2 going to sleep for 43 seconds 00000100:00100000:2.0:1761231219.966560:0:11662:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b74af89 pname:cluuid:pid:xid:nid:opc:job fsx:0995b904-d672-4cf9-8bdd-5554fffeedd5:11662:1846780606987264:192.168.201.104@tcp:36:fsx.0 00000080:00200000:2.0:1761231219.966587:0:11662:0:(file.c:1543:ll_io_set_mirror()) f0.fsxfile: desiginated mirror: 0 00000080:00200000:2.0:1761231219.966591:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:2.0:1761231219.966612:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231219.966627:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:2.0:1761231219.966643:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.966646:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000f006e797, osc lock 00000000bca714f2, flags 0x0 00000008:00010000:2.0:1761231219.966650:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231219.966651:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c3d1a1c5, osc lock 000000006ed9356c, flags 0x0 00010000:00010000:2.0:1761231219.966661:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.966670:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.966679:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.966686:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.966692:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.966699:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.966711:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 2/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.966715:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.966719:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231219.966721:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1761231219.967756:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dd4d5876 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606987328:192.168.201.104@tcp:10:fsx.0 00000020:00010000:2.0:1761231220.024650:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.024654:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.024656:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fb3cdbf2 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.024658:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.024665:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.024675:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.024677:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fb3cdbf2 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.024685:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.024717:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 167 00000080:00200000:2.0:1761231220.024721:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.024727:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.024732:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [167, 167] 00000008:00010000:2.0:1761231220.024738:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.024740:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000b1149c57, osc lock 00000000e44f139f, flags 0x0 00010000:00010000:2.0:1761231220.024746:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.024755:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (684032 688127) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.024763:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.024769:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.024775:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.025647:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.025655:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.025659:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000b1149c57 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.025663:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.025671:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.025685:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.025687:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000b1149c57 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.025696:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.025717:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 168 00000080:00200000:2.0:1761231220.025721:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.025725:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.025730:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [168, 168] 00000008:00010000:2.0:1761231220.025736:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.025737:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000b1149c57, osc lock 00000000e44f139f, flags 0x0 00010000:00010000:2.0:1761231220.025743:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.025751:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (688128 692223) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.025759:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.025765:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.025771:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.027979:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.027983:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.027985:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000f541e392 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.027989:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.027995:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.028005:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.028008:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000f541e392 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.028016:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.028043:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 170 00000080:00200000:2.0:1761231220.028048:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.028054:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.028060:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [170, 170] 00000008:00010000:2.0:1761231220.028066:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.028069:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000b1bef812, flags 0x0 00010000:00010000:2.0:1761231220.028075:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.028084:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (696320 700415) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.028092:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.028099:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.028105:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.029119:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.029124:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.029127:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.029130:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.029136:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.029146:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.029148:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.029156:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.029181:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 171 00000080:00200000:2.0:1761231220.029186:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.029191:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.029196:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [171, 171] 00000008:00010000:2.0:1761231220.029202:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.029204:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000b1bef812, flags 0x0 00010000:00010000:2.0:1761231220.029211:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.029218:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (700416 704511) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.029226:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.029244:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.029250:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.030348:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.030352:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.030354:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.030357:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.030364:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.030371:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.030372:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.030379:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.030411:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 172 00000080:00200000:2.0:1761231220.030415:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.030419:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.030423:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [172, 172] 00000008:00010000:2.0:1761231220.030428:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.030429:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000b1bef812, flags 0x0 00010000:00010000:2.0:1761231220.030435:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.030445:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (704512 708607) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.030450:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.030453:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.030456:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.031411:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.031416:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.031418:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.031422:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031430:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.031439:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.031441:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.031450:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.031594:0:11662:0:(file.c:5227:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef),name=f0.fsxfile 00000002:00010000:2.0:1761231220.031613:0:11662:0:(mdc_locks.c:1328:mdc_intent_lock()) (name: ,[0x200000401:0x64f6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1761231220.031622:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231220.031630:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1761231220.031641:0:11662:0:(dcache.c:293:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) 00000080:00010000:2.0:1761231220.031644:0:11662:0:(llite_internal.h:1711:ll_set_lock_data()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) for lock 0x311d04a022a02500 00000080:00010000:2.0:1761231220.031648:0:11662:0:(dcache.c:193:ll_intent_drop_lock()) releasing lock with cookie 0x311d04a022a02500 from it 0000000056434e97 00010000:00010000:2.0:1761231220.031650:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231220.031656:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1761231220.031668:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231220.031673:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231220.031682:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.031683:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000f006e797, osc lock 00000000b1bef812, flags 0x400001002 00000008:00010000:2.0:1761231220.031688:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.031689:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c3d1a1c5, osc lock 00000000e9a41835, flags 0x400001002 00010000:00010000:2.0:1761231220.031696:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031703:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031711:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031718:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031729:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031736:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031742:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031748:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.031759:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4315866 00000020:00010000:2.0:1761231220.031761:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.031764:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.031766:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000f006e797 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.031768:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c3d1a1c5 (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.031770:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.031772:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000f006e797 (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.031775:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c3d1a1c5 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.031780:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.031788:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=0=0x0(2) 00000080:00200000:2.0:1761231220.031792:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231220.031795:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231220.031800:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.031801:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000f006e797, osc lock 00000000e9a41835, flags 0x400001002 00000008:00010000:2.0:1761231220.031805:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.031806:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c3d1a1c5, osc lock 00000000b1bef812, flags 0x400001002 00010000:00010000:2.0:1761231220.031810:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031817:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031824:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031864:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031875:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031883:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031889:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.031895:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.031905:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4315866 00000020:00010000:2.0:1761231220.031907:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.031909:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.031911:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000f006e797 (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.031913:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c3d1a1c5 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.031915:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.031917:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000f006e797 (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.031920:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c3d1a1c5 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.031924:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00a00000:2.0:1761231220.032032:0:11662:0:(llite_mmap.c:603:ll_file_mmap()) VFS_Op: fid=[0x200000401:0x64f6:0x0] vma=00000000113e8c53 start=0x7fbc0644b000 end=0x7fbc06455000 vm_flags=0xfb 00000080:00200000:2.0:1761231220.032039:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231220.032042:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231220.032046:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.032047:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000f006e797, osc lock 00000000b1bef812, flags 0x400001002 00000008:00010000:2.0:1761231220.032049:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.032049:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c3d1a1c5, osc lock 00000000e9a41835, flags 0x400001002 00010000:00010000:2.0:1761231220.032053:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.032058:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.032062:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.032066:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.032072:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.032076:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.032080:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.032083:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.032088:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4315866 00000020:00010000:2.0:1761231220.032089:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.032090:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.032091:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000f006e797 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.032092:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c3d1a1c5 (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.032093:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.032094:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000f006e797 (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.032096:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c3d1a1c5 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.032099:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.032118:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 516 00000080:00200000:2.0:1761231220.032121:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.032124:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.032128:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [516, 516] 00000008:00010000:2.0:1761231220.032132:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.032133:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.032137:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.032142:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1064960 1069055) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.032146:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.032150:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.032153:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.033250:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.033256:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.033258:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.033262:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.033275:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.033284:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.033286:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.033296:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.033311:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.033317:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.033322:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [516, 516] 00000008:00010000:2.0:1761231220.033328:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.033330:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.033336:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.033344:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1064960 1069055) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.033351:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.033357:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.033363:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.033385:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.033414:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231220.033425:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.033427:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.033429:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.033430:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.033436:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.033444:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.033446:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.033453:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.033476:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 517 00000080:00200000:2.0:1761231220.033481:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.033486:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.033491:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [517, 517] 00000008:00010000:2.0:1761231220.033496:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.033498:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.033504:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.033512:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1069056 1073151) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.033519:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.033525:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.033531:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.034479:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.034484:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.034485:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.034488:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.034496:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.034503:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.034504:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.034511:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.034520:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.034524:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.034527:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [517, 517] 00000008:00010000:2.0:1761231220.034532:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.034533:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.034539:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.034547:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1069056 1073151) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.034555:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.034566:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.034573:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.034590:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.034603:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231220.034611:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.034613:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.034615:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.034618:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.034625:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.034633:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.034635:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.034642:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.034664:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 518 00000080:00200000:2.0:1761231220.034668:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.034673:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.034678:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [518, 518] 00000008:00010000:2.0:1761231220.034684:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.034686:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.034697:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.034705:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1073152 1077247) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.034713:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.034720:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.034726:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.035651:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.035656:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.035658:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.035662:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.035671:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.035680:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.035683:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.035691:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.035703:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.035708:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.035713:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [518, 518] 00000008:00010000:2.0:1761231220.035719:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.035721:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.035727:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.035735:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1073152 1077247) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.035742:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.035748:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.035754:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.035771:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.035784:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231220.035790:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.035792:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.035794:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.035797:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.035804:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.035812:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.035814:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.035820:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.035860:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 519 00000080:00200000:2.0:1761231220.035864:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.035870:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.035874:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [519, 519] 00000008:00010000:2.0:1761231220.035880:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.035882:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.035888:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.035895:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1077248 1081343) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.035903:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.035909:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.035914:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.036852:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.036857:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.036860:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.036864:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.036873:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.036883:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.036885:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.036892:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.036904:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.036908:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.036912:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [519, 519] 00000008:00010000:2.0:1761231220.036918:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.036920:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.036925:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.036933:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1077248 1081343) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.036940:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.036952:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.036958:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.036971:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.036980:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231220.036984:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.036986:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.036987:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.036989:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.036993:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.036997:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.036999:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.037003:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.037018:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 520 00000080:00200000:2.0:1761231220.037021:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.037025:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.037027:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [520, 520] 00000008:00010000:2.0:1761231220.037031:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.037033:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.037037:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.037041:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1081344 1085439) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.037046:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.037050:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.037053:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.037997:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.038002:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.038005:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.038010:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.038017:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.038024:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.038025:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.038032:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.038041:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.038047:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.038052:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [520, 520] 00000008:00010000:2.0:1761231220.038059:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.038061:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.038068:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.038075:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1081344 1085439) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.038083:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.038089:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.038095:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.038112:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.038126:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231220.038133:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.038136:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.038138:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.038140:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.038145:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.038150:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.038151:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.038156:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.038173:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 521 00000080:00200000:2.0:1761231220.038176:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.038179:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.038188:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [521, 521] 00000008:00010000:2.0:1761231220.038192:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.038193:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.038198:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.038203:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1085440 1089535) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.038207:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.038211:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.038214:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.039158:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.039161:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.039163:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.039166:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.039172:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.039178:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.039179:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.039186:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.039195:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.039203:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.039206:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [521, 521] 00000008:00010000:2.0:1761231220.039210:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.039212:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.039216:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.039221:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1085440 1089535) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.039225:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.039233:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.039236:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.039246:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.039255:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231220.039259:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.039260:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.039261:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.039263:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.039282:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.039290:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.039292:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.039299:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.039321:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 522 00000080:00200000:2.0:1761231220.039325:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.039330:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.039334:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [522, 522] 00000008:00010000:2.0:1761231220.039339:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.039341:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.039346:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.039353:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1089536 1093631) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.039360:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.039366:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.039372:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.040474:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.040479:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.040482:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.040487:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.040495:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.040503:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.040506:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.040514:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.040526:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.040531:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.040535:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [522, 522] 00000008:00010000:2.0:1761231220.040541:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.040543:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.040549:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.040556:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1089536 1093631) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.040563:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.040569:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.040575:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.040591:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.040603:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231220.040611:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.040613:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.040615:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.040618:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.040625:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.040633:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.040635:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.040642:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.040663:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 523 00000080:00200000:2.0:1761231220.040668:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.040674:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.040678:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [523, 523] 00000008:00010000:2.0:1761231220.040683:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.040684:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.040690:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.040697:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1093632 1097727) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.040704:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.040710:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.040716:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.041847:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.041852:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.041857:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.041864:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.041874:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.041884:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.041886:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.041894:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.041906:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.041912:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.041917:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [523, 523] 00000008:00010000:2.0:1761231220.041923:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.041925:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.041931:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.041939:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1093632 1097727) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.041948:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.041960:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.041967:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.041984:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.041997:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231220.042003:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.042006:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.042008:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.042010:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.042017:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.042024:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.042026:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.042033:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.042054:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 524 00000080:00200000:2.0:1761231220.042058:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.042064:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.042068:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [524, 524] 00000008:00010000:2.0:1761231220.042074:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.042076:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.042082:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.042089:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1097728 1101823) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.042097:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.042103:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.042109:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.043067:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.043072:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.043073:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.043076:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.043083:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.043089:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.043091:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.043098:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.043108:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.043112:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.043116:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [524, 524] 00000008:00010000:2.0:1761231220.043120:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.043122:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.043127:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.043132:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1097728 1101823) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.043137:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.043141:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.043144:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.043165:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.043174:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231220.043179:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.043180:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.043182:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.043183:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.043188:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.043192:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.043194:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.043198:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.043215:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 525 00000080:00200000:2.0:1761231220.043218:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.043221:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.043224:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [525, 525] 00000008:00010000:2.0:1761231220.043228:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.043229:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.043234:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.043239:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1101824 1105919) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.043243:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.043247:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.043250:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.044331:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.044335:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.044337:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.044340:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.044347:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.044354:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.044355:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.044362:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.044371:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.044375:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.044379:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [525, 525] 00000008:00010000:2.0:1761231220.044384:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.044385:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c0387933, osc lock 00000000e9a41835, flags 0x0 00010000:00010000:2.0:1761231220.044390:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.044395:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1101824 1105919) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.044415:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.044423:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 9/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.044427:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 8/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.044438:0:11662:0:(osc_cache.c:791:osc_extent_find()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.044446:0:11662:0:(vvp_io.c:1122:vvp_set_batch_dirty()) mapping 000000007084988a, count 1, dirtied 1 00000020:00010000:2.0:1761231220.044451:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.044452:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.044454:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c0387933 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:2.0:1761231220.044455:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.044459:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1761231220.044464:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.044465:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c0387933 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.044469:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.044480:0:11662:0:(file.c:4709:ll_fsync()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), start 2113536, end 2154495, datasync 1 00000080:00200000:2.0:1761231220.044484:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.044487:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1761231220.044493:0:11662:0:(osc_cache.c:1877:try_to_add_extent_for_io()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.044497:0:11662:0:(osc_cache.c:1877:try_to_add_extent_for_io()) ### extent: 00000000567b784e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.044501:0:11662:0:(osc_cache.c:1877:try_to_add_extent_for_io()) ### extent: 00000000aaf549f8 ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.044506:0:11662:0:(osc_cache.c:1084:osc_extent_make_ready()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.044536:0:11662:0:(osc_cache.c:1084:osc_extent_make_ready()) ### extent: 00000000567b784e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.044543:0:11662:0:(osc_cache.c:1084:osc_extent_make_ready()) ### extent: 00000000aaf549f8 ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.044563:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1900544 1904639) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00100000:2.0:1761231220.044623:0:11662:0:(osc_request.c:1944:osc_brw_prep_request()) brw rpc 0000000013e22019 - object 0x0:23051 offset 1040384<>2218714 00000008:00010000:2.0:1761231220.046863:0:2230:0:(osc_cache.c:843:osc_extent_finish()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.046907:0:2230:0:(osc_cache.c:503:__osc_extent_remove()) ### extent: 000000004250b56e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.046921:0:2230:0:(osc_cache.c:843:osc_extent_finish()) ### extent: 00000000567b784e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.046932:0:2230:0:(osc_cache.c:503:__osc_extent_remove()) ### extent: 00000000567b784e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.046942:0:2230:0:(osc_cache.c:843:osc_extent_finish()) ### extent: 00000000aaf549f8 ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231220.046963:0:2230:0:(osc_cache.c:503:__osc_extent_remove()) ### extent: 00000000aaf549f8 ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1761231220.046976:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013e22019 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606992128:192.168.201.104@tcp:4:fsx.0 00000080:00200000:2.0:1761231220.057917:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.057982:0:11662:0:(file.c:5227:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef),name=f0.fsxfile 00000002:00010000:2.0:1761231220.058000:0:11662:0:(mdc_locks.c:1328:mdc_intent_lock()) (name: ,[0x200000401:0x64f6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1761231220.058013:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231220.058023:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1761231220.058030:0:11662:0:(dcache.c:293:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) 00000080:00010000:2.0:1761231220.058032:0:11662:0:(llite_internal.h:1711:ll_set_lock_data()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) for lock 0x311d04a022a02500 00000080:00010000:2.0:1761231220.058037:0:11662:0:(dcache.c:193:ll_intent_drop_lock()) releasing lock with cookie 0x311d04a022a02500 from it 0000000056434e97 00010000:00010000:2.0:1761231220.058039:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231220.058044:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1761231220.058059:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231220.058065:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231220.058077:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.058080:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000f006e797, osc lock 00000000e9a41835, flags 0x400001002 00000008:00010000:2.0:1761231220.058083:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.058084:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c3d1a1c5, osc lock 00000000b1bef812, flags 0x400001002 00010000:00010000:2.0:1761231220.058092:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058097:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058104:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058108:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058115:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058119:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058126:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058130:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.058141:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4315866 00000020:00010000:2.0:1761231220.058142:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.058144:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.058145:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000f006e797 (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.058147:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c3d1a1c5 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.058149:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.058151:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000f006e797 (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.058154:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c3d1a1c5 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.058160:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.058168:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=0=0x0(2) 00000080:00200000:2.0:1761231220.058172:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231220.058174:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231220.058179:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.058181:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000f006e797, osc lock 00000000b1bef812, flags 0x400001002 00000008:00010000:2.0:1761231220.058184:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.058185:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c3d1a1c5, osc lock 00000000e9a41835, flags 0x400001002 00010000:00010000:2.0:1761231220.058190:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058197:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058205:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058211:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058218:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058222:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058226:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058229:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.058234:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4315866 00000020:00010000:2.0:1761231220.058235:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.058236:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.058237:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000f006e797 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.058239:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c3d1a1c5 (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.058240:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.058241:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000f006e797 (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.058242:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c3d1a1c5 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.058246:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.058255:0:11662:0:(file.c:5227:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef),name=f0.fsxfile 00000002:00010000:2.0:1761231220.058260:0:11662:0:(mdc_locks.c:1328:mdc_intent_lock()) (name: ,[0x200000401:0x64f6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1761231220.058263:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231220.058267:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1761231220.058271:0:11662:0:(dcache.c:293:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) 00000080:00010000:2.0:1761231220.058272:0:11662:0:(llite_internal.h:1711:ll_set_lock_data()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) for lock 0x311d04a022a02500 00000080:00010000:2.0:1761231220.058274:0:11662:0:(dcache.c:193:ll_intent_drop_lock()) releasing lock with cookie 0x311d04a022a02500 from it 0000000056434e97 00010000:00010000:2.0:1761231220.058275:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231220.058278:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1761231220.058283:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231220.058285:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231220.058289:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.058289:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000f006e797, osc lock 00000000e9a41835, flags 0x400001002 00000008:00010000:2.0:1761231220.058291:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.058292:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c3d1a1c5, osc lock 00000000b1bef812, flags 0x400001002 00010000:00010000:2.0:1761231220.058294:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058298:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058306:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058310:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058314:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058317:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058320:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058324:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.058329:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4315866 00000020:00010000:2.0:1761231220.058329:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.058331:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.058332:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000f006e797 (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.058333:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c3d1a1c5 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.058334:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.058336:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000f006e797 (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.058337:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c3d1a1c5 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.058340:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.058343:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=0=0x0(2) 00000080:00200000:2.0:1761231220.058345:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1761231220.058347:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:2.0:1761231220.058349:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.058350:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000f006e797, osc lock 00000000b1bef812, flags 0x400001002 00000008:00010000:2.0:1761231220.058352:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1761231220.058352:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c3d1a1c5, osc lock 00000000e9a41835, flags 0x400001002 00010000:00010000:2.0:1761231220.058364:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058368:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058375:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058378:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058383:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058386:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058390:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.058397:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1761231220.058416:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4315866 00000020:00010000:2.0:1761231220.058418:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:2.0:1761231220.058419:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.058420:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000f006e797 (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.058422:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000c3d1a1c5 (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:2.0:1761231220.058423:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.058424:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000f006e797 (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:2.0:1761231220.058426:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000c3d1a1c5 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:2.0:1761231220.058429:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:2.0:1761231220.058485:0:11662:0:(llite_lib.c:2191:ll_setattr_raw()) lustre: setattr inode [0x200000401:0x64f6:0x0](0000000047dd89ef) from 4315866 to 4451038, valid 2068, hsm_import 0 00010000:00010000:2.0:1761231220.058495:0:11662:0:(ldlm_request.c:1232:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231220.058505:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000008bf8410e/0x311d04a0229f468f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x64f6:0x0].0x0 bits 0x9/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb90906fe7 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231220.058532:0:11662:0:(ldlm_request.c:1279:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1761231220.058536:0:11662:0:(ldlm_request.c:1325:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1761231220.058537:0:11662:0:(ldlm_lock.c:216:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 000000004737b846/0x311d04a022a02500 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc2e856eb9093083f expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000020:00100000:2.0:1761231220.058551:0:11662:0:(genops.c:2339:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9001d2597000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1761231220.058555:0:11662:0:(client.c:742:ptlrpc_reassign_next_xid()) @@@ reassign xid req@00000000cf28a251 x1846780606992384/t0(0) o36->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761231220.058569:0:11662:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cf28a251 pname:cluuid:pid:xid:nid:opc:job fsx:0995b904-d672-4cf9-8bdd-5554fffeedd5:11662:1846780606992384:192.168.201.104@tcp:36:fsx.0 00000100:00100000:2.0:1761231220.058616:0:11662:0:(client.c:2500:ptlrpc_set_wait()) set 0000000089d178b2 going to sleep for 43 seconds 00000100:00100000:2.0:1761231220.059858:0:11662:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cf28a251 pname:cluuid:pid:xid:nid:opc:job fsx:0995b904-d672-4cf9-8bdd-5554fffeedd5:11662:1846780606992384:192.168.201.104@tcp:36:fsx.0 00000080:00200000:2.0:1761231220.059885:0:11662:0:(file.c:1543:ll_io_set_mirror()) f0.fsxfile: desiginated mirror: 0 00000080:00200000:2.0:1761231220.059888:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:2.0:1761231220.059899:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1761231220.059912:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:2.0:1761231220.059920:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.059922:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000f006e797, osc lock 00000000e9a41835, flags 0x0 00000008:00010000:2.0:1761231220.059925:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1761231220.059925:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000c3d1a1c5, osc lock 00000000b1bef812, flags 0x0 00010000:00010000:2.0:1761231220.059934:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.059943:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.059951:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.059957:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,2 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.059962:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.059976:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.059982:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.059989:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.059994:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/0,2 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.059999:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1761231220.060057:0:2230:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761231220.060069:0:2230:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606992448:192.168.201.104@tcp:10:fsx.0 00000100:00100000:2.0:1761231220.060150:0:2230:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a4b6d86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606992512:192.168.201.104@tcp:10:fsx.0 00000100:00100000:2.0:1761231220.061091:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a4b6d86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606992512:192.168.201.104@tcp:10:fsx.0 00000100:00100000:2.0:1761231220.061117:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606992448:192.168.201.104@tcp:10:fsx.0 00000008:00010000:0.0:1761231220.087068:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.087070:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 000000004a70038e, flags 0x0 00010000:00010000:0.0:1761231220.087074:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.087079:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (2252800 2256895) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.087084:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.087088:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.087091:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.088042:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.088046:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.088049:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.088052:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.088060:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.088068:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.088070:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.088078:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.088107:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 1063 00000080:00200000:0.0:1761231220.088110:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.088116:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.088122:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [1063, 1063] 00000008:00010000:0.0:1761231220.088128:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.088130:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 000000004a70038e, flags 0x0 00010000:00010000:0.0:1761231220.088136:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.088143:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (2256896 2260991) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.088150:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.088157:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.088162:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.089079:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.089082:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.089084:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.089087:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.089093:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.089103:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.089104:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.089111:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.089131:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 1064 00000080:00200000:0.0:1761231220.089135:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.089139:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.089142:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [1064, 1064] 00000008:00010000:0.0:1761231220.089147:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.089149:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 000000004a70038e, flags 0x0 00010000:00010000:0.0:1761231220.089154:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.089159:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (2260992 2265087) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.089164:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.089168:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.089171:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.090101:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.090106:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.090109:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.090113:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.090121:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.090129:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.090132:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.090140:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.090170:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 1065 00000080:00200000:0.0:1761231220.090174:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.090179:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.090184:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [1065, 1065] 00000008:00010000:0.0:1761231220.090190:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.090192:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 000000004a70038e, flags 0x0 00010000:00010000:0.0:1761231220.090198:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.090206:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (2265088 2269183) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.090214:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.090221:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.090225:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.091166:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.091170:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.091172:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.091174:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.091180:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.091186:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.091187:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.091193:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.091211:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 1066 00000080:00200000:0.0:1761231220.091214:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.091217:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.091221:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [1066, 1066] 00000008:00010000:0.0:1761231220.091225:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.091226:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 000000004a70038e, flags 0x0 00010000:00010000:0.0:1761231220.091231:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.091236:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (2269184 2273279) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.091241:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.091248:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.091252:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.092148:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.092152:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.092155:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.092159:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.092166:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.092174:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.092176:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.092183:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.092204:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 1067 00000080:00200000:0.0:1761231220.092208:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.092213:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.092217:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [1067, 1067] 00000008:00010000:0.0:1761231220.092223:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.092225:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 000000004a70038e, flags 0x0 00010000:00010000:0.0:1761231220.092231:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.092244:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (2273280 2277375) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.092252:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.092258:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.092263:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.093112:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.093115:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.093117:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.093119:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.093124:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.093130:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.093131:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.093137:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.093155:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 1068 00000080:00200000:0.0:1761231220.093158:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.093161:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.093165:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [1068, 1068] 00000008:00010000:0.0:1761231220.093169:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.093170:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 000000004a70038e, flags 0x0 00010000:00010000:0.0:1761231220.093175:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.093180:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (2277376 2281471) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.093185:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.093189:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.093192:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.094078:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.094082:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.094085:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.094088:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.094096:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.094109:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.094111:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.094120:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.094145:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 1069 00000080:00200000:0.0:1761231220.094149:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.094155:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.094160:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [1069, 1069] 00000008:00010000:0.0:1761231220.094166:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.094168:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 000000004a70038e, flags 0x0 00010000:00010000:0.0:1761231220.094174:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.094182:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (2281472 2285567) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.094189:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.094194:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.094200:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.095087:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.095090:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.095092:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.095095:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.095100:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.095106:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.095107:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.095113:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.095132:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 1070 00000080:00200000:0.0:1761231220.095135:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.095139:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.095143:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [1070, 1070] 00000008:00010000:0.0:1761231220.095147:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.095148:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 000000004a70038e, flags 0x0 00010000:00010000:0.0:1761231220.095152:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.095158:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (2285568 2289663) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.095162:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.095166:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 7/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.095169:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.096014:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.096017:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.096019:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000905dce88/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.096022:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096026:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.096032:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.096033:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.096039:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.096102:0:11662:0:(file.c:5227:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef),name=f0.fsxfile 00000002:00010000:0.0:1761231220.096118:0:11662:0:(mdc_locks.c:1328:mdc_intent_lock()) (name: ,[0x200000401:0x64f6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1761231220.096126:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 0000000072cb30c9/0x311d04a022a02507 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb90930854 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1761231220.096132:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 0000000072cb30c9/0x311d04a022a02507 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb90930854 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00010000:0.0:1761231220.096139:0:11662:0:(dcache.c:293:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) 00000080:00010000:0.0:1761231220.096141:0:11662:0:(llite_internal.h:1711:ll_set_lock_data()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) for lock 0x311d04a022a02507 00000080:00010000:0.0:1761231220.096144:0:11662:0:(dcache.c:193:ll_intent_drop_lock()) releasing lock with cookie 0x311d04a022a02507 from it 0000000056434e97 00010000:00010000:0.0:1761231220.096145:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 0000000072cb30c9/0x311d04a022a02507 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xc2e856eb90930854 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1761231220.096152:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 0000000072cb30c9/0x311d04a022a02507 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb90930854 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1761231220.096161:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1761231220.096165:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:0.0:1761231220.096170:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.096171:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000e00c92cd, osc lock 000000004a70038e, flags 0x400001002 00000008:00010000:0.0:1761231220.096174:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.096174:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000009121c97a, osc lock 00000000758abf97, flags 0x400001002 00010000:00010000:0.0:1761231220.096178:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096183:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096187:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096190:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096197:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096201:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096208:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096211:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1761231220.096217:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4680994 00000020:00010000:0.0:1761231220.096219:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.096220:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.096221:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000e00c92cd (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.096222:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000009121c97a (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.096224:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.096225:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000e00c92cd (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.096226:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000009121c97a (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.096230:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.096235:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=0=0x0(2) 00000080:00200000:0.0:1761231220.096237:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1761231220.096239:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:0.0:1761231220.096242:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.096243:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000e00c92cd, osc lock 00000000758abf97, flags 0x400001002 00000008:00010000:0.0:1761231220.096245:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.096245:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000009121c97a, osc lock 000000004a70038e, flags 0x400001002 00010000:00010000:0.0:1761231220.096248:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096252:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096256:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096260:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096264:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096267:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096270:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096274:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1761231220.096278:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4680994 00000020:00010000:0.0:1761231220.096279:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.096280:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.096282:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000e00c92cd (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.096283:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000009121c97a (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.096284:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.096285:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000e00c92cd (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.096286:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000009121c97a (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.096289:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00a00000:0.0:1761231220.096329:0:11662:0:(llite_mmap.c:603:ll_file_mmap()) VFS_Op: fid=[0x200000401:0x64f6:0x0] vma=0000000005783bb6 start=0x7fbc06444000 end=0x7fbc06455000 vm_flags=0xf9 00000080:00200000:0.0:1761231220.096334:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1761231220.096336:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:0.0:1761231220.096339:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.096339:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000e00c92cd, osc lock 000000004a70038e, flags 0x400001002 00000008:00010000:0.0:1761231220.096341:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.096342:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000009121c97a, osc lock 00000000758abf97, flags 0x400001002 00010000:00010000:0.0:1761231220.096344:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096349:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096353:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096356:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096361:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096365:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096368:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096375:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1761231220.096383:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4680994 00000020:00010000:0.0:1761231220.096385:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.096387:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.096389:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000e00c92cd (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.096391:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000009121c97a (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.096393:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.096395:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000e00c92cd (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.096408:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000009121c97a (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.096414:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.096441:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 967 00000080:00200000:0.0:1761231220.096445:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.096449:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.096454:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [967, 967] 00000008:00010000:0.0:1761231220.096460:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.096461:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.096467:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096474:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1863680 1867775) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096481:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096493:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.096499:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.097509:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.097512:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.097514:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.097519:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.097527:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.097535:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.097538:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.097546:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.097568:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 968 00000080:00200000:0.0:1761231220.097572:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.097578:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.097583:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [968, 968] 00000008:00010000:0.0:1761231220.097589:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.097591:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.097597:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.097610:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1867776 1871871) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.097618:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.097625:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.097630:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.098726:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.098731:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.098733:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.098738:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.098747:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.098757:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.098759:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.098767:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.098788:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 969 00000080:00200000:0.0:1761231220.098791:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.098796:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.098800:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [969, 969] 00000008:00010000:0.0:1761231220.098804:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.098806:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.098811:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.098817:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1871872 1875967) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.098823:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.098842:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.098848:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.099688:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.099691:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.099693:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.099695:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.099701:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.099710:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.099711:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.099717:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.099735:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 970 00000080:00200000:0.0:1761231220.099738:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.099741:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.099744:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [970, 970] 00000008:00010000:0.0:1761231220.099749:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.099750:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.099754:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.099759:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1875968 1880063) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.099764:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.099768:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.099771:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.100658:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.100661:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.100663:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.100666:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.100672:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.100678:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.100680:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.100686:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.100704:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 971 00000080:00200000:0.0:1761231220.100706:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.100710:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.100714:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [971, 971] 00000008:00010000:0.0:1761231220.100718:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.100720:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.100724:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.100730:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1880064 1884159) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.100735:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.100738:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.100741:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.101654:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.101657:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.101659:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.101662:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.101667:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.101673:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.101674:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.101680:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.101697:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 972 00000080:00200000:0.0:1761231220.101700:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.101704:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.101708:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [972, 972] 00000008:00010000:0.0:1761231220.101712:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.101713:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.101718:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.101723:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1884160 1888255) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.101727:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.101735:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.101738:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.102636:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.102639:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.102641:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.102643:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.102648:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.102653:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.102655:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.102660:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.102679:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 973 00000080:00200000:0.0:1761231220.102681:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.102685:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.102689:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [973, 973] 00000008:00010000:0.0:1761231220.102693:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.102694:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.102698:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.102708:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1888256 1892351) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.102713:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.102716:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.102719:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.103504:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.103508:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.103509:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.103512:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.103517:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.103522:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.103524:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.103529:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.103546:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 974 00000080:00200000:0.0:1761231220.103548:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.103552:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.103555:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [974, 974] 00000008:00010000:0.0:1761231220.103559:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.103560:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.103564:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.103569:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1892352 1896447) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.103574:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.103578:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.103581:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.104483:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.104487:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.104489:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.104492:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.104500:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.104513:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.104525:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.104537:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.104560:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 975 00000080:00200000:0.0:1761231220.104564:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.104569:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.104574:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [975, 975] 00000008:00010000:0.0:1761231220.104580:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.104581:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.104587:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.104595:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1896448 1900543) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.104602:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.104607:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.104610:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.105471:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.105475:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.105477:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.105481:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.105488:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.105497:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.105499:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.105506:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.105525:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 976 00000080:00200000:0.0:1761231220.105528:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.105531:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.105534:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [976, 976] 00000008:00010000:0.0:1761231220.105539:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.105540:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.105544:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.105549:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1900544 1904639) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.105554:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.105558:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.105561:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.106417:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.106421:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.106423:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.106426:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.106433:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.106441:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.106443:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.106451:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.106473:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 977 00000080:00200000:0.0:1761231220.106477:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.106482:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.106486:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [977, 977] 00000008:00010000:0.0:1761231220.106492:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.106494:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.106500:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.106508:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1904640 1908735) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.106515:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.106524:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.106527:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.107418:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.107422:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.107424:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.107427:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.107435:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.107443:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.107445:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.107453:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.107476:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 978 00000080:00200000:0.0:1761231220.107479:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.107484:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.107490:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [978, 978] 00000008:00010000:0.0:1761231220.107495:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.107497:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.107503:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.107516:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1908736 1912831) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.107523:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.107529:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.107534:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.108347:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.108351:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.108354:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.108358:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.108365:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.108374:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.108376:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.108383:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.108421:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 979 00000080:00200000:0.0:1761231220.108426:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.108432:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.108437:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [979, 979] 00000008:00010000:0.0:1761231220.108443:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.108444:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.108451:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.108458:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1912832 1916927) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.108465:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.108471:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.108476:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.109509:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.109513:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.109515:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.109518:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.109524:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.109534:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.109535:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.109542:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.109563:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 980 00000080:00200000:0.0:1761231220.109566:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.109570:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.109573:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [980, 980] 00000008:00010000:0.0:1761231220.109578:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.109580:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.109585:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.109590:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1916928 1921023) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.109595:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.109599:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.109602:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.110552:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.110555:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.110557:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.110560:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.110565:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.110571:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.110572:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.110578:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.110604:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 981 00000080:00200000:0.0:1761231220.110608:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.110613:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.110618:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [981, 981] 00000008:00010000:0.0:1761231220.110625:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.110626:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.110632:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.110640:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1921024 1925119) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.110647:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.110653:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.110659:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.111611:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.111615:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.111618:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.111621:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.111629:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.111638:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.111640:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.111650:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.111670:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 982 00000080:00200000:0.0:1761231220.111673:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.111676:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.111680:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [982, 982] 00000008:00010000:0.0:1761231220.111684:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.111686:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.111690:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.111695:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1925120 1929215) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.111700:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.111708:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.111712:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.112614:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.112617:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.112619:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.112621:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.112625:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.112631:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.112632:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.112638:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.112656:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 983 00000080:00200000:0.0:1761231220.112659:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.112662:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.112666:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [983, 983] 00000008:00010000:0.0:1761231220.112670:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.112671:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.112675:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.112684:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1929216 1933311) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.112689:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.112693:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.112696:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.113695:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.113700:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.113703:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000fa49f574 (00000000cb18b166/1) at cl_lock_cancel():151 00010000:00010000:0.0:1761231220.113707:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.113715:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1761231220.113723:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000d04151d5 (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.113725:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000fa49f574 (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.113733:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.113842:0:11662:0:(file.c:5227:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef),name=f0.fsxfile 00000002:00010000:0.0:1761231220.113868:0:11662:0:(mdc_locks.c:1328:mdc_intent_lock()) (name: ,[0x200000401:0x64f6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1761231220.113878:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 0000000072cb30c9/0x311d04a022a02507 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb90930854 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1761231220.113891:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 0000000072cb30c9/0x311d04a022a02507 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb90930854 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00010000:0.0:1761231220.113902:0:11662:0:(dcache.c:293:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) 00000080:00010000:0.0:1761231220.113905:0:11662:0:(llite_internal.h:1711:ll_set_lock_data()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) for lock 0x311d04a022a02507 00000080:00010000:0.0:1761231220.113909:0:11662:0:(dcache.c:193:ll_intent_drop_lock()) releasing lock with cookie 0x311d04a022a02507 from it 0000000056434e97 00010000:00010000:0.0:1761231220.113911:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 0000000072cb30c9/0x311d04a022a02507 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xc2e856eb90930854 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1761231220.113918:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 0000000072cb30c9/0x311d04a022a02507 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb90930854 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1761231220.113931:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1761231220.113937:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:0.0:1761231220.113946:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.113947:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000e00c92cd, osc lock 00000000758abf97, flags 0x400001002 00000008:00010000:0.0:1761231220.113952:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.113953:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000009121c97a, osc lock 000000004a70038e, flags 0x400001002 00010000:00010000:0.0:1761231220.113961:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.113968:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.113976:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.113983:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.113993:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114000:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114007:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114018:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1761231220.114028:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4680994 00000020:00010000:0.0:1761231220.114031:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.114034:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.114036:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000e00c92cd (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.114039:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000009121c97a (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.114041:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.114043:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000e00c92cd (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.114046:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000009121c97a (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.114052:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.114060:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=0=0x0(2) 00000080:00200000:0.0:1761231220.114063:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1761231220.114067:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:0.0:1761231220.114071:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.114073:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000e00c92cd, osc lock 000000004a70038e, flags 0x400001002 00000008:00010000:0.0:1761231220.114076:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.114077:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000009121c97a, osc lock 00000000758abf97, flags 0x400001002 00010000:00010000:0.0:1761231220.114081:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114089:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114096:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114102:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114110:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114117:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114124:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114130:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1761231220.114138:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4680994 00000020:00010000:0.0:1761231220.114140:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.114142:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.114145:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000e00c92cd (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.114147:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000009121c97a (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.114149:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.114151:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000e00c92cd (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.114154:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000009121c97a (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.114158:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.114168:0:11662:0:(file.c:5227:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef),name=f0.fsxfile 00000002:00010000:0.0:1761231220.114175:0:11662:0:(mdc_locks.c:1328:mdc_intent_lock()) (name: ,[0x200000401:0x64f6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1761231220.114179:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 0000000072cb30c9/0x311d04a022a02507 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb90930854 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1761231220.114186:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 0000000072cb30c9/0x311d04a022a02507 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb90930854 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00010000:0.0:1761231220.114193:0:11662:0:(dcache.c:293:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) 00000080:00010000:0.0:1761231220.114195:0:11662:0:(llite_internal.h:1711:ll_set_lock_data()) setting l_data to inode [0x200000401:0x64f6:0x0](0000000047dd89ef) for lock 0x311d04a022a02507 00000080:00010000:0.0:1761231220.114198:0:11662:0:(dcache.c:193:ll_intent_drop_lock()) releasing lock with cookie 0x311d04a022a02507 from it 0000000056434e97 00010000:00010000:0.0:1761231220.114199:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 0000000072cb30c9/0x311d04a022a02507 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xc2e856eb90930854 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1761231220.114205:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9001d2597000 lock: 0000000072cb30c9/0x311d04a022a02507 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x64f6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc2e856eb90930854 expref: -99 pid: 11662 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1761231220.114218:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1761231220.114221:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:0.0:1761231220.114226:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.114228:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000e00c92cd, osc lock 00000000758abf97, flags 0x400001002 00000008:00010000:0.0:1761231220.114231:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.114232:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000009121c97a, osc lock 000000004a70038e, flags 0x400001002 00010000:00010000:0.0:1761231220.114236:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114244:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114251:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114258:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114266:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114272:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114279:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114290:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1761231220.114298:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4680994 00000020:00010000:0.0:1761231220.114300:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.114302:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.114304:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000e00c92cd (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.114306:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000009121c97a (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.114308:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.114310:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000e00c92cd (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.114313:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000009121c97a (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.114318:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.114323:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=0=0x0(2) 00000080:00200000:0.0:1761231220.114326:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1761231220.114329:0:11662:0:(glimpse.c:90:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x64f6:0x0] 00000008:00010000:0.0:1761231220.114333:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.114334:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000e00c92cd, osc lock 000000004a70038e, flags 0x400001002 00000008:00010000:0.0:1761231220.114337:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1761231220.114338:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 000000009121c97a, osc lock 00000000758abf97, flags 0x400001002 00010000:00010000:0.0:1761231220.114342:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114350:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114356:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114363:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114371:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114377:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114383:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114389:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/0,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1761231220.114412:0:11662:0:(file.c:1495:ll_merge_attr()) [0x200000401:0x64f6:0x0] updating i_size 4680994 00000020:00010000:0.0:1761231220.114414:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) release lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_release():234 00000020:00010000:0.0:1761231220.114416:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.114418:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 00000000e00c92cd (00000000905dce88/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.114421:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) cancel lock: 000000009121c97a (00000000cb18b166/1) at cl_lock_cancel():151 00000020:00010000:0.0:1761231220.114423:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000380f2dee (00000000495e9d75/0) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.114425:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 00000000e00c92cd (00000000495e9d75/1) at cl_lock_fini():84 00000020:00010000:0.0:1761231220.114428:0:11662:0:(cl_lock.c:53:__cl_lock_trace()) destroy lock: 000000009121c97a (00000000495e9d75/1) at cl_lock_fini():84 00000080:00200000:0.0:1761231220.114432:0:11662:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0 00000080:00200000:0.0:1761231220.114466:0:11662:0:(file.c:4585:ll_file_seek()) VFS Op:inode=[0x200000401:0x64f6:0x0](0000000047dd89ef), to=1932244=0x1d7bd4(0) 00000080:00200008:0.0:1761231220.114474:0:11662:0:(file.c:2028:ll_file_read_iter()) file f0.fsxfile:[0x200000401:0x64f6:0x0], ppos: 1932244, count: 44669 00000080:00200000:0.0:1761231220.114494:0:11662:0:(rw.c:1868:ll_readpage()) fast read pgno: 471 00000080:00200000:0.0:1761231220.114500:0:11662:0:(file.c:1671:ll_file_io_generic()) f0.fsxfile: read ppos: 1932244, count: 44669 00000080:00200000:0.0:1761231220.114503:0:11662:0:(file.c:1543:ll_io_set_mirror()) f0.fsxfile: desiginated mirror: 0 00000020:00200000:0.0:1761231220.114508:0:11662:0:(cl_io.c:216:cl_io_rw_init()) header@000000008978f1bb[0x4, 327, [0x200000401:0x64f6:0x0] hash] 00000020:00200000:0.0:1761231220.114510:0:11662:0:(cl_io.c:216:cl_io_rw_init()) io range: 1 [1932244, 1976913) 0 0 00000080:00200000:0.0:1761231220.114512:0:11662:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1761231220.114520:0:11662:0:(lov_io.c:996:lov_io_rw_iter_init()) stripe: 1 chunk: [1932244, 1976913) 1976913, 44669 00020000:00200000:0.0:1761231220.114523:0:11662:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1761231220.114532:0:11662:0:(vvp_io.c:222:vvp_io_one_lock_index()) lock: 0 [471, 482] 00000008:00010000:0.0:1761231220.114537:0:11662:0:(lustre_osc.h:181:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1761231220.114539:0:11662:0:(osc_lock.c:1241:osc_lock_init()) ### lock 00000000fa49f574, osc lock 00000000758abf97, flags 0x0 00010000:00010000:0.0:1761231220.114544:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114552:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (880640 929791) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 3/0,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114559:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114566:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114572:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1761231220.114578:0:11662:0:(vvp_io.c:829:vvp_io_read_start()) f0.fsxfile: read [1932244, 1976913) 00000080:00200000:0.0:1761231220.114587:0:11662:0:(vvp_io.c:886:vvp_io_read_start()) count:44669 iocb pos:1932244 00000080:00200000:0.0:1761231220.114590:0:11662:0:(rw.c:1924:ll_readpage()) pgno:471, beyond read end_index:483 00010000:00010000:0.0:1761231220.114605:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114613:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (884736 888831) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114621:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 8/2,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114627:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 8/2,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114633:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/2,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114950:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/2,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114959:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114971:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114975:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1048576 1052671) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114980:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114984:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.114987:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.116170:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.116178:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.116187:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.116191:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (1048576 1052671) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.116196:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 8/2,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.116199:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 8/2,1 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.116202:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/2,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.117649:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 7/2,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.117656:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 6/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.117667:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.117673:0:11662:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (2097152 2101247) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.117681:0:11662:0:(ldlm_lock.c:766:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.117685:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 6/1,1 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.117689:0:11662:0:(ldlm_lock.c:909:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.118191:0:11662:0:(ldlm_lock.c:820:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 5/1,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1761231220.118198:0:11662:0:(ldlm_lock.c:902:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 4/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231220.442938:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761231222.555021:0:2231:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2231:1846780606997312:192.168.201.104@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761231222.555100:0:2231:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0995b904-d672-4cf9-8bdd-5554fffeedd5:2231:1846780606997440:192.168.201.104@tcp:400:kworker/u8:2.0 00010000:00010000:2.0:1761231225.562988:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761231227.675078:0:2231:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761231227.675092:0:2231:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0995b904-d672-4cf9-8bdd-5554fffeedd5:2231:1846780606997632:192.168.201.104@tcp:400:kworker/u8:2.0 00000100:00000400:2.0:1761231229.722943:0:2231:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1761231222/real 1761231222] req@00000000fc01c41c x1846780606997312/t0(0) o400->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 224/224 e 0 to 1 dl 1761231229 ref 1 fl Rpc:XNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761231229.722996:0:2231:0:(client.c:2339:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@00000000fc01c41c x1846780606997312/t0(0) o400->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 224/224 e 0 to 1 dl 1761231229 ref 1 fl Rpc:XNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761231229.723009:0:2231:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1761231229.723015:0:2231:0:(import.c:206:ptlrpc_set_import_discon()) 166-1: MGC192.168.201.104@tcp: Connection to MGS (at 192.168.201.104@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1761231229.732770:0:2231:0:(import.c:212:ptlrpc_set_import_discon()) import MGS@192.168.201.104@tcp for MGC192.168.201.104@tcp not replayable, auto-deactivating 00000100:00080000:2.0:1761231229.732775:0:2231:0:(import.c:157:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:2.0:1761231229.732781:0:2231:0:(client.c:3295:ptlrpc_abort_inflight()) @@@ inflight req@00000000fc01c41c x1846780606997312/t0(0) o400->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 224/224 e 0 to 1 dl 1761231229 ref 1 fl Rpc:XNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761231229.732790:0:2231:0:(client.c:3295:ptlrpc_abort_inflight()) @@@ inflight req@00000000f87d0618 x1846780606997568/t0(0) o400->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1761231234 ref 1 fl Rpc:NQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 10000000:01000000:2.0:1761231229.732798:0:2231:0:(mgc_request.c:1332:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1761231229.732801:0:2231:0:(pinger.c:215:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:2.0:1761231229.732802:0:2231:0:(mgc_request.c:1332:mgc_import_event()) import event 0x808002 00000100:00080000:2.0:1761231229.732803:0:2231:0:(import.c:434:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000400:2.0:1761231229.732822:0:2231:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1761231222/real 1761231222] req@00000000408f088c x1846780606997440/t0(0) o400->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1761231229 ref 1 fl Rpc:XNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231229.732849:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761231229.732853:0:2231:0:(client.c:2339:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@00000000408f088c x1846780606997440/t0(0) o400->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1761231229 ref 1 fl Rpc:XNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231229.732856:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231229.732859:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231229.732861:0:2231:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:00080000:0.0:1761231229.732866:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 140 00000100:02000400:2.0:1761231229.732867:0:2231:0:(import.c:200:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffff9001d2597000: Connection to lustre-OST0000 (at 192.168.201.104@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1761231229.732875:0:2231:0:(import.c:434:ptlrpc_pinger_force()) lustre-OST0000_UUID: waking up pinger s:DISCONN 00000100:00080000:0.0:1761231229.732876:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231229.732896:0:2231:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2231:1846780606997312:192.168.201.104@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761231229.732907:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780606997824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231229.732935:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780606997824:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761231229.733021:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231229.733024:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231229.733026:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231229.733029:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 1056 00000100:00080000:0.0:1761231229.733032:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231229.733038:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231229.733047:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780606997952/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231229.738813:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231229.738818:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231229.738820:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231229.738825:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 1056 00000100:00080000:0.0:1761231229.738847:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231229.738857:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231229.738876:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780606998016/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761231229.738925:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780606998016:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761231230.682942:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231234.779045:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231234.842992:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231234.843001:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1761231234.843059:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231234.843062:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1761231234.843064:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231234.843067:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000080:00200000:2.0:1761231235.354976:0:11821:0:(vvp_io.c:1820:vvp_io_init()) [0x200000401:0x64f6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1761231235.354992:0:11821:0:(lov_io.c:825:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1761231235.355020:0:11821:0:(osc_cache.c:1877:try_to_add_extent_for_io()) ### extent: 00000000cf09994e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231235.355037:0:11821:0:(osc_cache.c:1084:osc_extent_make_ready()) ### extent: 00000000cf09994e ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 2/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231235.355090:0:11821:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (2547712 2551807) ns: lustre-OST0000-osc-ffff9001d2597000 lock: 0000000020790460/0x311d04a0229f4696 lrc: 3/0,0 mode: PW/PW res: [0x5a0b:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90906ffc expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00100000:2.0:1761231235.355196:0:11821:0:(osc_request.c:1944:osc_brw_prep_request()) brw rpc 0000000013e22019 - object 0x0:23051 offset 2547712<>2583842 00000008:00010000:2.0:1761231235.355231:0:11821:0:(osc_cache.c:1877:try_to_add_extent_for_io()) ### extent: 00000000fbe58d8d ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1761231235.355239:0:11821:0:(osc_cache.c:1084:osc_extent_make_ready()) ### extent: 00000000fbe58d8d ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 4/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1761231235.355264:0:11821:0:(ldlm_lock.c:1494:ldlm_lock_match_with_skip()) ### matched (24576 28671) ns: lustre-OST0001-osc-ffff9001d2597000 lock: 000000001866f6a2/0x311d04a0229f469d lrc: 5/1,0 mode: PW/PW res: [0x5a0c:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xc2e856eb90907003 expref: -99 pid: 11662 timeout: 0 lvb_type: 1 00000008:00100000:2.0:1761231235.355302:0:11821:0:(osc_request.c:1944:osc_brw_prep_request()) brw rpc 00000000dd4d5876 - object 0x0:23052 offset 0<>53248 00000080:00200000:2.0:1761231235.355323:0:11821:0:(vvp_io.c:313:vvp_io_fini()) [0x200000401:0x64f6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0 00010000:00010000:2.0:1761231235.802959:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761231235.867596:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606997760:192.168.201.104@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761231235.867607:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606997504:192.168.201.104@tcp:400:kworker/u8:2.0 00000100:00080000:0.0:1761231235.867645:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231235.867650:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231235.867688:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231235.867692:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231235.867697:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231235.867706:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231235.867710:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231235.867717:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4107 00000100:00080000:0.0:1761231235.867719:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 6s 00000100:00080000:0.0:1761231235.867724:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231235.867733:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231235.867752:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780606998336/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761231235.867775:0:2231:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0995b904-d672-4cf9-8bdd-5554fffeedd5:2231:1846780606997440:192.168.201.104@tcp:400:kworker/u8:2.0 00000100:00080000:0.0:1761231235.867782:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231235.867785:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231235.867787:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231235.867791:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231235.867794:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231235.867800:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4107 00000100:00080000:0.0:1761231235.867802:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 6s 00000100:00080000:0.0:1761231235.867808:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1761231235.867810:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606997696:192.168.201.104@tcp:400:kworker/u8:2.0 00000008:00100000:0.0:1761231235.867819:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231235.867864:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780606998400/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231235.867957:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231235.867962:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231235.867964:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231235.867971:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4107 00000100:00080000:2.0:1761231235.867976:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 6s 00000100:00080000:2.0:1761231235.867980:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231235.867995:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780606998464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00010000:00010000:0.0:1761231239.898995:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231240.922992:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231240.986948:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231240.986958:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1761231240.987035:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231240.987039:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1761231240.987041:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231240.987045:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1761231246.043057:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231246.106945:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231246.106956:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231246.107022:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231246.107026:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231246.107030:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231246.107035:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231246.107113:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231246.107119:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231246.107123:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231246.107139:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4113 00000100:00080000:0.0:1761231246.107149:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 11s 00000100:00080000:0.0:1761231246.107158:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231246.107177:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780606998656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231246.107186:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231246.107188:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231246.107190:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231246.107195:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4113 00000100:00080000:0.0:1761231246.107200:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 11s 00000100:00080000:0.0:1761231246.107202:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231246.107208:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231246.107218:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780606998720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231246.107224:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231246.107226:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231246.107227:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231246.107230:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4113 00000100:00080000:0.0:1761231246.107232:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 11s 00000100:00080000:0.0:1761231246.107236:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231246.107241:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231246.107251:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780606998784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231251.227048:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231251.227058:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231251.227108:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231251.227111:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231251.227113:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231251.227115:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1761231256.347054:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231256.347063:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1761231256.347111:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231256.347114:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1761231256.347116:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231256.347117:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1761231256.347157:0:2230:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761231256.347172:0:2230:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606998912:192.168.201.104@tcp:400:kworker/u8:0.0 00000100:00000400:2.0:1761231257.371003:0:2230:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231240/real 1761231257] req@0000000017386078 x1846780606998528/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231284 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761231257.380563:0:2230:0:(client.c:2339:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@0000000017386078 x1846780606998528/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231284 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761231257.380572:0:2230:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1761231257.380575:0:2230:0:(import.c:200:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff9001d2597000: Connection to lustre-MDT0000 (at 192.168.201.104@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1761231257.384290:0:2230:0:(import.c:434:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00100000:2.0:1761231257.384306:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606998528:192.168.201.104@tcp:400:kworker/u8:2.0 00000100:00080000:0.0:1761231257.384330:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231257.384335:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231257.384337:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231257.384340:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231257.384342:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231257.384347:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 140 00000100:00080000:0.0:1761231257.384353:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231257.384361:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231257.384383:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780606998976/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231257.384399:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231257.384402:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231257.384403:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231257.384405:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1761231257.384434:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780606998976:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00080000:0.0:1761231262.490998:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231262.491010:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231262.491014:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231262.491019:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231262.491024:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761231262.491024:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1761231246/real 0] req@0000000001b2f813 x1846780606998784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231262 ref 2 fl Rpc:XNr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231262.491032:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231262.491037:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231262.491046:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1761231262.491092:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780606998784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231262 ref 2 fl UnregRPC:XNU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761231262.491112:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1761231246/real 0] req@00000000b8f2be78 x1846780606998720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231262 ref 2 fl Rpc:XNr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761231262.491133:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780606998720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231262 ref 2 fl UnregRPC:XNU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761231262.491149:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1761231246/real 0] req@0000000099514377 x1846780606998656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231262 ref 2 fl Rpc:XNr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761231262.491170:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780606998656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231262 ref 2 fl UnregRPC:XNU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00000400:0.0:1761231264.539090:0:2229:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1761231220/real 1761231220] req@00000000ca077941 x1846780606997120/t0(0) o3->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:6/4 lens 520/440 e 0 to 1 dl 1761231264 ref 2 fl Rpc:XQr/0/ffffffff rc 0/-1 job:'fsx.0' 00000100:00080000:2.0:1761231264.539154:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761231264.539187:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761231264.539197:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780606998784:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00080000:2.0:1761231264.539216:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761231264.539225:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761231264.539229:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780606998720:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00080000:2.0:1761231264.539241:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761231264.539251:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761231264.539256:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780606998656:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761231264.539308:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231264.539315:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231264.539320:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231264.539327:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4123 00000100:00080000:2.0:1761231264.539333:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 16s 00000100:00080000:2.0:1761231264.539343:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231264.539380:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780606999040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231264.539413:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231264.539417:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231264.539419:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231264.539423:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4123 00000100:00080000:2.0:1761231264.539425:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 16s 00000100:00080000:2.0:1761231264.539429:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231264.539439:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231264.539453:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780606999104/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231264.539465:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231264.539468:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231264.539470:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231264.539480:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4123 00000100:00080000:2.0:1761231264.539485:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 16s 00000100:00080000:2.0:1761231264.539494:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231264.539501:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231264.539517:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780606999168/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231264.563799:0:2229:0:(import.c:234:ptlrpc_set_import_discon()) osc: import 00000000a1f01947 already not connected (conn 8, was 4): CONNECTING 00000100:00000400:0.0:1761231264.563818:0:2229:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1761231220/real 1761231220] req@000000007dc14a18 x1846780606997184/t0(0) o3->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:6/4 lens 504/440 e 0 to 1 dl 1761231264 ref 2 fl Rpc:XQr/0/ffffffff rc 0/-1 job:'fsx.0' 00000100:00080000:0.0:1761231264.563893:0:2229:0:(import.c:234:ptlrpc_set_import_discon()) osc: import 00000000a1f01947 already not connected (conn 8, was 4): CONNECTING 00000100:00000400:0.0:1761231264.563903:0:2229:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1761231220/real 1761231220] req@000000007d19768b x1846780606996992/t0(0) o3->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:6/4 lens 488/440 e 0 to 1 dl 1761231264 ref 2 fl Rpc:XQr/0/ffffffff rc 0/-1 job:'fsx.0' 00000100:00080000:0.0:1761231264.563928:0:2229:0:(import.c:234:ptlrpc_set_import_discon()) osc: import 000000008bcc4531 already not connected (conn 8, was 4): CONNECTING 00000100:00000400:0.0:1761231264.563936:0:2229:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1761231220/real 1761231220] req@00000000d3d91900 x1846780606996928/t0(0) o3->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:6/4 lens 504/440 e 0 to 1 dl 1761231264 ref 2 fl Rpc:XQr/0/ffffffff rc 0/-1 job:'fsx.0' 00000100:00080000:0.0:1761231264.563962:0:2229:0:(import.c:234:ptlrpc_set_import_discon()) osc: import 000000008bcc4531 already not connected (conn 8, was 4): CONNECTING 00000100:00000400:0.0:1761231264.563982:0:2229:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231246/real 1761231264] req@0000000023ab3a3f x1846780606998592/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231290 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231264.563997:0:2229:0:(client.c:2339:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@0000000023ab3a3f x1846780606998592/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231290 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231264.564011:0:2229:0:(import.c:234:ptlrpc_set_import_discon()) mdc: import 000000002dbdfc93 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:0.0:1761231264.564017:0:2229:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0995b904-d672-4cf9-8bdd-5554fffeedd5:2229:1846780606998592:192.168.201.104@tcp:400:kworker/u8:2.0 00000100:00000400:2.0:1761231266.587039:0:2230:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1761231222/real 1761231222] req@000000001f8e688e x1846780606997376/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231266 ref 1 fl Rpc:XNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761231266.587113:0:2230:0:(client.c:2339:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@000000001f8e688e x1846780606997376/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231266 ref 1 fl Rpc:XNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761231266.587128:0:2230:0:(import.c:234:ptlrpc_set_import_discon()) mdc: import 000000002dbdfc93 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:2.0:1761231266.587164:0:2230:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2230:1846780606997376:192.168.201.104@tcp:400:kworker/u8:2.0 00000100:00000400:0.0:1761231268.635204:0:2229:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231251/real 1761231268] req@00000000271a9d8e x1846780606998848/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231295 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231268.661362:0:2229:0:(client.c:2339:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@00000000271a9d8e x1846780606998848/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231295 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231268.661378:0:2229:0:(import.c:234:ptlrpc_set_import_discon()) mdc: import 000000002dbdfc93 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:0.0:1761231268.661388:0:2229:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0995b904-d672-4cf9-8bdd-5554fffeedd5:2229:1846780606998848:192.168.201.104@tcp:400:kworker/u8:0.0 00000100:00080000:0.0:1761231269.659022:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231269.659034:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231269.659038:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231269.659041:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231269.659044:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231269.659048:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231269.659050:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231269.659052:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1761231272.731069:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761231272.731106:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761231272.731120:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780606998976:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00080000:0.0:1761231272.731186:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231272.731204:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231272.731207:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231272.731211:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231272.731214:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231272.731220:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4134 00000100:00080000:0.0:1761231272.731223:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 6s 00000100:00080000:0.0:1761231272.731230:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231272.731240:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231272.731267:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780606999232/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231272.731293:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231272.731298:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231272.731302:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231272.731307:0:11073:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1761231272.731339:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780606999232:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00000400:0.0:1761231273.626952:0:2229:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1761231229/real 1761231229] req@00000000275aeee5 x1846780606997888/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231273 ref 1 fl Rpc:XNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231273.626980:0:2229:0:(client.c:2339:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@00000000275aeee5 x1846780606997888/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231273 ref 1 fl Rpc:XNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231273.626986:0:2229:0:(import.c:234:ptlrpc_set_import_discon()) mdc: import 000000002dbdfc93 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:0.0:1761231273.627000:0:2229:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0995b904-d672-4cf9-8bdd-5554fffeedd5:2229:1846780606997888:192.168.201.104@tcp:400:kworker/u8:2.0 00010000:00010000:2.0:1761231276.763016:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1761231278.875129:0:2231:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1761231234/real 1761231234] req@0000000091106a97 x1846780606998080/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231278 ref 1 fl Rpc:XNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:2.0:1761231278.898267:0:2231:0:(client.c:2339:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@0000000091106a97 x1846780606998080/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231278 ref 1 fl Rpc:XNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:2.0:1761231278.898278:0:2231:0:(import.c:234:ptlrpc_set_import_discon()) mdc: import 000000002dbdfc93 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:2.0:1761231278.898301:0:2231:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0995b904-d672-4cf9-8bdd-5554fffeedd5:2231:1846780606998080:192.168.201.104@tcp:400:kworker/u8:0.0 00000100:00000400:0.0:1761231279.898992:0:2228:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1761231235/real 1761231235] req@000000005a096598 x1846780606998272/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231279 ref 1 fl Rpc:XNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231279.899029:0:2228:0:(client.c:2339:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@000000005a096598 x1846780606998272/t0(0) o400->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1761231279 ref 1 fl Rpc:XNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231279.899038:0:2228:0:(import.c:234:ptlrpc_set_import_discon()) mdc: import 000000002dbdfc93 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:0.0:1761231279.899051:0:2228:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:0995b904-d672-4cf9-8bdd-5554fffeedd5:2228:1846780606998272:192.168.201.104@tcp:400:kworker/u8:2.0 00000100:00080000:0.0:1761231280.923144:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231280.923161:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231280.923169:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231280.923185:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4142 00000100:00080000:0.0:1761231280.923192:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231280.923202:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231280.923251:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780606999296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231280.923280:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231280.923286:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231280.923293:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231280.923308:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231280.923312:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231280.923324:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4142 00000100:00080000:0.0:1761231280.923329:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231280.923342:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231280.923360:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231280.923405:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780606999360/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231280.923433:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231280.923437:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231280.923440:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231280.923446:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4142 00000100:00080000:0.0:1761231280.923451:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231280.923454:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231280.923458:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231280.923468:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780606999424/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231280.923482:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231280.923485:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1761231281.883074:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231286.042977:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231286.042990:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231286.042996:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231286.043001:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231286.043004:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231286.043008:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1761231286.043010:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231286.043014:0:11821:0:(pinger.c:265:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1761231287.003015:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231304.410987:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231305.434962:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231308.059109:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231305/real 1761231308] req@00000000659d78e3 x1846780606999744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231326 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231308.059140:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780606999744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231326 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231308.059151:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231308.059155:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231308.059160:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780606999744:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231308.059166:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231297/real 1761231308] req@00000000b72feabb x1846780606999616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231322 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231308.059176:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780606999616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231322 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231308.059181:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231308.059200:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231308.059203:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780606999616:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231308.059209:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231297/real 1761231308] req@00000000d07717c4 x1846780606999680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231322 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231308.059216:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780606999680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231322 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231308.059220:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231308.059224:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231308.059226:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780606999680:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231308.059229:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231297/real 1761231308] req@00000000db10af3c x1846780606999552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231322 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231308.059236:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780606999552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231322 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231308.059240:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231308.059245:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231308.059247:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780606999552:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231308.059364:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780606999808:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231308.059425:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231308/real 1761231308] req@00000000db10af3c x1846780606999808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231333 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231308.059440:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780606999808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231333 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231308.059450:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231308.059454:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231308.059458:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780606999808:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231308.059474:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780606999872:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231308.059502:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231308/real 1761231308] req@00000000d07717c4 x1846780606999872/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231333 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231308.059509:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780606999872/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231333 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231308.059514:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231308.059515:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231308.059517:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780606999872:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231308.059520:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780606999936:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231308.059532:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231308/real 1761231308] req@00000000b72feabb x1846780606999936/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231333 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231308.059537:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780606999936/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231333 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231308.059542:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231308.059543:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231308.059545:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780606999936:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231308.059552:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000000:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231308.059563:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231308/real 1761231308] req@00000000659d78e3 x1846780607000000/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231333 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231308.059569:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607000000/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231333 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231308.059574:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231308.059575:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231308.059577:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000000:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231308.506961:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231309.530991:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231310.554952:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231313.179153:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607000064:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231313.179256:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000128:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231313.179292:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000192:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231313.179314:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000256:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231314.650967:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231315.675009:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231316.251299:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231313/real 1761231316] req@00000000a58ca11e x1846780607000192/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231338 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231316.251325:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607000192/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231338 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231316.251335:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231316.251339:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231316.251342:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000192:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231316.251347:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231313/real 1761231316] req@0000000083f16a6b x1846780607000256/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231338 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231316.251358:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607000256/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231338 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231316.251368:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231316.251370:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231316.251373:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000256:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231316.251379:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231313/real 1761231316] req@00000000f87d0618 x1846780607000064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231338 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231316.251391:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607000064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231338 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231316.251399:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231316.251402:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231316.251405:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607000064:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231316.251411:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231313/real 1761231316] req@00000000c7e4e432 x1846780607000128/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231338 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231316.251422:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607000128/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231338 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231316.251429:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231316.251432:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231316.251434:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000128:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00080000:2.0:1761231318.299032:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231318.299044:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231318.299048:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231318.299058:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4190 00000100:00080000:2.0:1761231318.299061:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231318.299067:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231318.299093:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607000320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231318.299114:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231318.299116:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231318.299117:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231318.299120:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4190 00000100:00080000:2.0:1761231318.299122:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231318.299124:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231318.299130:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231318.299140:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607000384/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231318.299150:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231318.299153:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231318.299154:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1761231318.299161:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607000320:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:2.0:1761231318.299162:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4190 00000100:00080000:2.0:1761231318.299166:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231318.299171:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231318.299177:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231318.299188:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607000448/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231318.299198:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231318.299201:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231318.299202:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231318.299206:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4190 00000100:00080000:2.0:1761231318.299208:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231318.299210:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231318.299214:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231318.299225:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607000512/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231318.299267:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000384:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231318.299301:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000448:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231318.299319:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000512:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231319.323435:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231318/real 1761231319] req@00000000f38f27df x1846780607000448/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231343 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231319.323479:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607000448/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231343 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231319.323498:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231319.323504:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231319.323512:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000448:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231319.323522:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231318/real 1761231319] req@0000000017386078 x1846780607000512/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231343 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231319.323556:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607000512/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231343 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231319.323571:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231319.323574:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231319.323580:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000512:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231319.323589:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231318/real 1761231319] req@0000000091106a97 x1846780607000320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231343 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231319.323607:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607000320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231343 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231319.323620:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231319.323624:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231319.323628:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607000320:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231319.323638:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231318/real 1761231319] req@0000000064978143 x1846780607000384/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231343 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231319.323657:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607000384/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231343 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231319.323670:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231319.323675:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231319.323681:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000384:192.168.201.104@tcp:38:kworker/u8:0.0 00010000:00010000:2.0:1761231319.771082:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231320.795044:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231323.419125:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607000576:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231323.419223:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000640:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231323.419266:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000704:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231323.419283:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000768:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:2.0:1761231324.891078:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231325.915090:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231326.491415:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231323/real 1761231326] req@0000000031e043f4 x1846780607000640/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231348 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231326.491463:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607000640/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231348 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231326.491483:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231326.491490:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231326.491498:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000640:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231326.491510:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231323/real 1761231326] req@000000007361665d x1846780607000704/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231348 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231326.491547:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607000704/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231348 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231326.491563:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231326.491568:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231326.491575:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000704:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231326.491586:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231323/real 1761231326] req@0000000086ccb527 x1846780607000768/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231348 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231326.491606:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607000768/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231348 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231326.491622:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231326.491626:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231326.491632:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000768:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231326.491642:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231323/real 1761231326] req@00000000bd684ccc x1846780607000576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231348 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231326.491662:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607000576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231348 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231326.491676:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231326.491679:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231326.491682:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607000576:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231328.539103:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607000832:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231328.539187:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231328/real 1761231328] req@0000000018a8f5da x1846780607000832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231353 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231328.539201:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607000832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231353 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231328.539208:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231328.539210:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231328.539213:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607000832:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231328.539218:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000896:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231328.539242:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231328/real 1761231328] req@000000005a9bdefd x1846780607000896/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231353 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231328.539252:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607000896/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231353 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231328.539260:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231328.539262:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231328.539264:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000896:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231328.539290:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000960:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231328.539316:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231328/real 1761231328] req@0000000088473b3e x1846780607000960/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231353 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231328.539325:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607000960/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231353 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231328.539332:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231328.539334:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231328.539337:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607000960:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231328.539342:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001024:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231328.539364:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231328/real 1761231328] req@000000004e3f612a x1846780607001024/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231353 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231328.539375:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607001024/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231353 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231328.539383:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231328.539385:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231328.539387:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001024:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:2.0:1761231330.011117:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231331.035055:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231333.658984:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231333.658993:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231333.658996:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231333.659004:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4206 00000100:00080000:2.0:1761231333.659007:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231333.659014:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231333.659051:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607001088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231333.659072:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231333.659075:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231333.659076:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231333.659078:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4206 00000100:00080000:2.0:1761231333.659080:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231333.659082:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231333.659089:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231333.659098:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607001152/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231333.659104:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231333.659107:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231333.659108:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231333.659110:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4206 00000100:00080000:2.0:1761231333.659111:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231333.659113:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231333.659116:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231333.659120:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607001088:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:2.0:1761231333.659124:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607001216/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231333.659131:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231333.659133:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231333.659133:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231333.659136:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4206 00000100:00080000:2.0:1761231333.659137:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231333.659139:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231333.659142:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231333.659148:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607001280/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231333.659194:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001152:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231333.659218:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001216:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231333.659235:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001280:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:2.0:1761231335.130965:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231336.155003:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231336.731193:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231333/real 1761231336] req@0000000064978143 x1846780607001216/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231358 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231336.731219:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607001216/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231358 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231336.731231:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231336.731234:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231336.731238:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001216:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231336.731244:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231333/real 1761231336] req@0000000091106a97 x1846780607001280/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231358 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231336.731252:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607001280/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231358 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231336.731257:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231336.731259:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231336.731260:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001280:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231336.731263:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231333/real 1761231336] req@0000000094135f44 x1846780607001088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231358 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231336.731270:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607001088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231358 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231336.731274:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231336.731275:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231336.731276:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607001088:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231336.731280:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231333/real 1761231336] req@00000000c069b612 x1846780607001152/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231358 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231336.731286:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607001152/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231358 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231336.731290:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231336.731292:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231336.731293:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001152:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00080000:2.0:1761231338.779008:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231338.779018:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231338.779022:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231338.779033:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4211 00000100:00080000:2.0:1761231338.779036:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231338.779044:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231338.779077:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607001344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231338.779106:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231338.779111:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231338.779112:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231338.779117:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4211 00000100:00080000:2.0:1761231338.779119:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231338.779123:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231338.779132:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231338.779142:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607001408/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231338.779154:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231338.779157:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1761231338.779159:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607001344:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:2.0:1761231338.779160:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231338.779166:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4211 00000100:00080000:2.0:1761231338.779169:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231338.779171:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231338.779174:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231338.779184:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607001472/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231338.779190:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231338.779192:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231338.779193:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231338.779195:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4211 00000100:00080000:2.0:1761231338.779197:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231338.779198:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231338.779201:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231338.779207:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607001536/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231338.779246:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231338/real 1761231338] req@0000000017386078 x1846780607001344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231363 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231338.779260:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607001344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231363 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231338.779266:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231338.779269:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231338.779271:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607001344:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231338.779289:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001408:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231338.779305:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231338/real 1761231338] req@00000000f38f27df x1846780607001408/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231363 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231338.779312:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607001408/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231363 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231338.779316:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231338.779318:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231338.779320:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001408:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231338.779323:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001472:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231338.779335:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231338/real 1761231338] req@00000000c069b612 x1846780607001472/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231363 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231338.779341:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607001472/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231363 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231338.779346:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231338.779347:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231338.779349:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001472:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231338.779352:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001536:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231338.779363:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231338/real 1761231338] req@0000000094135f44 x1846780607001536/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231363 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231338.779368:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607001536/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231363 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231338.779373:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231338.779374:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231338.779376:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001536:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:2.0:1761231340.251023:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231341.275002:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231343.899122:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607001600:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231343.899197:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231343/real 1761231343] req@000000003ad149cc x1846780607001600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231368 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231343.899209:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607001600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231368 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231343.899216:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231343.899218:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231343.899220:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607001600:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231343.899225:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001664:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231343.899239:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231343/real 1761231343] req@00000000daa39bb1 x1846780607001664/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231368 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231343.899245:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607001664/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231368 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231343.899253:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231343.899255:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231343.899257:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001664:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231343.899260:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001728:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231343.899271:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231343/real 1761231343] req@000000004e3f612a x1846780607001728/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231368 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231343.899277:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607001728/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231368 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231343.899282:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231343.899283:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231343.899285:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001728:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231343.899302:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001792:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231343.899314:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231343/real 1761231343] req@0000000088473b3e x1846780607001792/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231368 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231343.899321:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607001792/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231368 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231343.899325:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231343.899326:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231343.899328:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001792:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:2.0:1761231345.371025:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231346.394976:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231349.018986:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231349.018995:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231349.018999:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231349.019008:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4221 00000100:00080000:2.0:1761231349.019010:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231349.019016:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231349.019046:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607001856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231349.019068:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231349.019070:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231349.019071:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231349.019074:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4221 00000100:00080000:2.0:1761231349.019075:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231349.019077:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231349.019084:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231349.019094:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607001920/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231349.019100:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231349.019103:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231349.019103:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231349.019106:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4221 00000100:00080000:2.0:1761231349.019107:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231349.019109:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231349.019112:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231349.019117:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607001856:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:2.0:1761231349.019123:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607001984/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231349.019129:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231349.019132:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231349.019133:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231349.019136:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4221 00000100:00080000:2.0:1761231349.019138:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231349.019140:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231349.019142:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231349.019148:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607002048/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231349.019197:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231349/real 1761231349] req@0000000091106a97 x1846780607001856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231374 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231349.019214:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607001856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231374 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231349.019224:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231349.019227:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231349.019231:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607001856:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231349.019239:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001920:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231349.019269:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231349/real 1761231349] req@0000000064978143 x1846780607001920/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231374 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231349.019278:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607001920/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231374 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231349.019283:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231349.019284:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231349.019286:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001920:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231349.019308:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001984:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231349.019337:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231349/real 1761231349] req@0000000094135f44 x1846780607001984/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231374 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231349.019349:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607001984/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231374 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231349.019356:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231349.019359:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231349.019362:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607001984:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231349.019368:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002048:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231349.019394:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231349/real 1761231349] req@00000000c069b612 x1846780607002048/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231374 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231349.019401:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607002048/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231374 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231349.019406:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231349.019408:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231349.019410:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002048:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:2.0:1761231350.491006:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231351.515009:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231354.139146:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607002112:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231354.139279:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002176:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231354.139303:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002240:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231354.139329:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002304:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231355.611010:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231357.211484:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231354/real 1761231357] req@0000000086ccb527 x1846780607002304/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231379 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231357.211529:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607002304/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231379 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231357.211548:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231357.211556:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231357.211563:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002304:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231357.211572:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231354/real 1761231357] req@000000005a9bdefd x1846780607002112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231379 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231357.211586:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607002112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231379 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231357.211594:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231357.211598:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231357.211602:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607002112:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231357.211609:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231354/real 1761231357] req@0000000018a8f5da x1846780607002176/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231379 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231357.211622:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607002176/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231379 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231357.211630:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231357.211633:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231357.211636:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002176:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231357.211644:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231354/real 1761231357] req@00000000bd684ccc x1846780607002240/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231379 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231357.211655:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607002240/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231379 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231357.211663:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231357.211667:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231357.211672:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002240:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231357.659077:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231359.259008:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231359.259012:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231359.259022:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4231 00000100:00080000:1.0:1761231359.259025:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231359.259031:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231359.259066:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607002368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231359.259090:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231359.259092:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231359.259093:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231359.259096:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4231 00000100:00080000:1.0:1761231359.259098:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231359.259100:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231359.259106:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231359.259113:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607002432/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231359.259119:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231359.259121:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231359.259122:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231359.259125:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4231 00000100:00080000:1.0:1761231359.259126:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231359.259128:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231359.259131:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231359.259138:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607002496/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231359.259144:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607002368:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761231359.259145:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231359.259152:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231359.259154:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231359.259157:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4231 00000100:00080000:1.0:1761231359.259158:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231359.259160:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231359.259162:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231359.259169:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607002560/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231359.259227:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231359/real 1761231359] req@00000000659d78e3 x1846780607002368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231384 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231359.259244:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607002368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231384 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231359.259254:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231359.259256:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231359.259260:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607002368:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231359.259267:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002432:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231359.259292:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231359/real 1761231359] req@00000000b72feabb x1846780607002432/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231384 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231359.259302:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607002432/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231384 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231359.259309:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231359.259311:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231359.259314:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002432:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231359.259355:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002496:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231359.259384:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231359/real 1761231359] req@00000000d07717c4 x1846780607002496/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231384 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231359.259403:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607002496/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231384 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231359.259411:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231359.259413:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231359.259417:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002496:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231359.259422:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002560:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231359.259446:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231359/real 1761231359] req@00000000db10af3c x1846780607002560/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231384 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231359.259457:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607002560/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231384 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231359.259465:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231359.259467:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231359.259470:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002560:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231360.731006:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231361.755003:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231362.779008:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231364.379186:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607002624:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231364.379313:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231364/real 1761231364] req@000000007361665d x1846780607002624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231389 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231364.379329:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607002624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231389 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231364.379337:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231364.379340:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231364.379344:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607002624:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231364.379350:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002688:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231364.379374:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231364/real 1761231364] req@0000000031e043f4 x1846780607002688/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231389 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231364.379385:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607002688/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231389 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231364.379393:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231364.379395:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231364.379398:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002688:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231364.379405:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002752:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231364.379431:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231364/real 1761231364] req@00000000c7e4e432 x1846780607002752/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231389 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231364.379442:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607002752/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231389 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231364.379451:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231364.379454:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231364.379457:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002752:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231364.379486:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002816:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231364.379504:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231364/real 1761231364] req@00000000f87d0618 x1846780607002816/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231389 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231364.379515:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607002816/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231389 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231364.379521:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231364.379522:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231364.379524:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002816:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231365.851080:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231367.899092:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231369.499147:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607002880:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231369.499267:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231369/real 1761231369] req@0000000083f16a6b x1846780607002880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231394 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231369.499280:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607002880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231394 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231369.499294:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231369.499296:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231369.499299:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607002880:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231369.499305:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002944:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231369.499323:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231369/real 1761231369] req@00000000a58ca11e x1846780607002944/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231394 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231369.499330:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607002944/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231394 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231369.499335:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231369.499337:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231369.499339:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607002944:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231369.499362:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003008:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231369.499390:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231369/real 1761231369] req@000000001f8e688e x1846780607003008/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231394 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231369.499404:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607003008/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231394 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231369.499413:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231369.499415:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231369.499419:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003008:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231369.499426:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003072:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231369.499455:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231369/real 1761231369] req@0000000090b40c66 x1846780607003072/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231394 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231369.499469:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607003072/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231394 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231369.499478:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231369.499480:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231369.499483:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003072:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231370.971089:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231373.019043:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231374.619215:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607003136:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231374.619297:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231374/real 1761231374] req@00000000408f088c x1846780607003136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231399 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231374.619310:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607003136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231399 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231374.619318:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231374.619320:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231374.619323:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607003136:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231374.619328:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003200:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231374.619343:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231374/real 1761231374] req@000000002ce934d3 x1846780607003200/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231399 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231374.619351:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607003200/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231399 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231374.619356:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231374.619357:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231374.619359:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003200:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231374.619362:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003264:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231374.619374:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231374/real 1761231374] req@000000006572b591 x1846780607003264/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231399 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231374.619381:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607003264/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231399 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231374.619386:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231374.619388:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231374.619390:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003264:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231374.619411:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003328:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231374.619424:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231374/real 1761231374] req@00000000fc01c41c x1846780607003328/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231399 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231374.619431:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607003328/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231399 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231374.619436:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231374.619437:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231374.619438:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003328:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231376.091025:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231378.139145:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231379.739285:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607003392:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231379.739452:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231379/real 1761231379] req@00000000fc01c41c x1846780607003392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231404 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231379.739476:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607003392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231404 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231379.739490:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231379.739493:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231379.739498:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607003392:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231379.739507:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003456:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231379.739538:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231379/real 1761231379] req@000000006572b591 x1846780607003456/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231404 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231379.739551:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607003456/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231404 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231379.739560:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231379.739562:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231379.739565:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003456:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231379.739600:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003520:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231379.739626:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231379/real 1761231379] req@000000002ce934d3 x1846780607003520/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231404 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231379.739638:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607003520/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231404 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231379.739647:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231379.739649:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231379.739652:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003520:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231379.739659:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003584:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231379.739679:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231379/real 1761231379] req@00000000408f088c x1846780607003584/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231404 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231379.739691:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607003584/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231404 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231379.739700:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231379.739702:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231379.739706:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003584:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231381.211158:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231383.259120:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231384.859287:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607003648:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231384.859414:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231384/real 1761231384] req@0000000090b40c66 x1846780607003648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231409 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231384.859441:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607003648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231409 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231384.859459:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231384.859465:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231384.859471:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607003648:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231384.859484:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003712:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231384.859526:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231384/real 1761231384] req@000000001f8e688e x1846780607003712/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231409 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231384.859539:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607003712/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231409 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231384.859548:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231384.859551:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231384.859554:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003712:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231384.859583:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003776:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231384.859608:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231384/real 1761231384] req@00000000a58ca11e x1846780607003776/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231409 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231384.859621:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607003776/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231409 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231384.859630:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231384.859632:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231384.859635:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003776:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231384.859641:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003840:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231384.859661:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231384/real 1761231384] req@0000000083f16a6b x1846780607003840/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231409 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231384.859672:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607003840/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231409 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231384.859681:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231384.859683:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231384.859686:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003840:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231386.331207:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231388.379158:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231389.979260:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607003904:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231389.979390:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003968:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231389.979429:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607004032:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231389.979449:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607004096:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231391.451127:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231393.051405:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231389/real 1761231393] req@0000000031e043f4 x1846780607004032/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231414 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231393.051454:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607004032/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231414 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231393.051475:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231393.051482:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231393.051489:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607004032:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231393.051500:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231389/real 1761231393] req@000000007361665d x1846780607004096/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231414 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231393.051519:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607004096/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231414 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231393.051533:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231393.051537:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231393.051541:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607004096:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231393.051551:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231389/real 1761231393] req@00000000f87d0618 x1846780607003904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231414 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231393.051571:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607003904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231414 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231393.051585:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231393.051590:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231393.051594:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607003904:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231393.051605:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231389/real 1761231393] req@00000000c7e4e432 x1846780607003968/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231414 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231393.051639:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607003968/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231414 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231393.051653:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231393.051658:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231393.051665:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607003968:192.168.201.104@tcp:38:kworker/u8:2.0 00010000:00010000:0.0:1761231393.499200:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231395.099047:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231395.099061:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231395.099066:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231395.099079:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4267 00000100:00080000:1.0:1761231395.099083:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231395.099091:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231395.099134:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607004160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231395.099168:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231395.099172:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231395.099174:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231395.099179:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4267 00000100:00080000:1.0:1761231395.099182:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231395.099185:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231395.099194:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231395.099209:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607004224/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231395.099222:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231395.099227:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231395.099229:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231395.099233:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4267 00000100:00080000:1.0:1761231395.099235:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231395.099239:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1761231395.099246:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607004160:192.168.201.104@tcp:250:kworker/u8:0.0 00000008:00100000:1.0:1761231395.099247:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231395.099265:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607004288/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231395.099280:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231395.099284:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231395.099286:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231395.099292:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4267 00000100:00080000:1.0:1761231395.099295:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231395.099299:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231395.099304:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231395.099318:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607004352/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231395.099370:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231395/real 1761231395] req@00000000db10af3c x1846780607004160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231420 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231395.099395:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607004160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231420 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231395.099407:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231395.099411:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231395.099415:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607004160:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231395.099424:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607004224:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231395.099453:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231395/real 1761231395] req@00000000d07717c4 x1846780607004224/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231420 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231395.099465:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607004224/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231420 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231395.099474:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231395.099476:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231395.099480:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607004224:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231395.099509:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607004288:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231395.099534:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231395/real 1761231395] req@00000000b72feabb x1846780607004288/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231420 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231395.099546:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607004288/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231420 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231395.099554:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231395.099556:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231395.099559:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607004288:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231395.099578:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607004352:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231395.099626:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231395/real 1761231395] req@00000000659d78e3 x1846780607004352/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231420 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231395.099647:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607004352/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231420 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231395.099661:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231395.099665:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231395.099671:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607004352:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231396.571144:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231400.218973:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231400.218987:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231400.218991:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231400.219003:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4272 00000100:00080000:0.0:1761231400.219008:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231400.219017:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231400.219059:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607004416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231400.219093:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231400.219097:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231400.219099:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231400.219104:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4272 00000100:00080000:0.0:1761231400.219106:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231400.219110:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231400.219120:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231400.219137:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607004480/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231400.219157:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231400.219161:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231400.219163:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231400.219169:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4272 00000100:00080000:0.0:1761231400.219172:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231400.219175:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231400.219181:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231400.219192:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607004544/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231400.219202:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231400.219205:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231400.219207:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231400.219212:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4272 00000100:00080000:0.0:1761231400.219214:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231400.219219:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231400.219224:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231400.219239:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607004608/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00010000:00010000:0.0:1761231401.691088:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231405.339043:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231405.339057:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231405.339063:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231405.339076:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4277 00000100:00080000:1.0:1761231405.339081:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231405.339092:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231405.339137:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607004672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231405.339175:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231405.339179:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231405.339181:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231405.339186:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4277 00000100:00080000:1.0:1761231405.339189:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231405.339193:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231405.339204:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231405.339219:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607004736/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231405.339236:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231405.339246:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231405.339248:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231405.339253:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4277 00000100:00080000:1.0:1761231405.339255:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231405.339258:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231405.339264:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231405.339281:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607004800/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231405.339291:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231405.339294:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231405.339296:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231405.339301:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4277 00000100:00080000:1.0:1761231405.339303:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231405.339307:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231405.339312:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231405.339326:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607004864/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00010000:00010000:0.0:1761231406.811158:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231410.459169:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231410.459183:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231410.459187:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231410.459201:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4282 00000100:00080000:2.0:1761231410.459206:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231410.459214:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231410.459258:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607004928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231410.459294:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231410.459299:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231410.459301:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231410.459306:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4282 00000100:00080000:2.0:1761231410.459308:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231410.459312:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231410.459324:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231410.459339:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607004992/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231410.459351:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231410.459355:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231410.459357:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231410.459361:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4282 00000100:00080000:2.0:1761231410.459363:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231410.459369:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231410.459378:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231410.459395:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607005056/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231410.459406:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231410.459409:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231410.459412:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231410.459416:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4282 00000100:00080000:2.0:1761231410.459419:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231410.459422:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231410.459427:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231410.459440:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607005120/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00010000:00010000:0.0:1761231411.931117:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231415.579278:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607005184:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231415.579414:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231415/real 1761231415] req@00000000bd684ccc x1846780607005184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231440 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231415.579437:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607005184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231440 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231415.579449:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231415.579453:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231415.579458:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607005184:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231415.579467:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005248:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231415.579532:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231415/real 1761231415] req@0000000018a8f5da x1846780607005248/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231440 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231415.579546:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607005248/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231440 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231415.579563:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231415.579566:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231415.579569:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005248:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231415.579599:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005312:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231415.579625:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231415/real 1761231415] req@000000005a9bdefd x1846780607005312/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231440 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231415.579637:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607005312/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231440 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231415.579646:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231415.579648:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231415.579651:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005312:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231415.579657:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005376:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231415.579677:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231415/real 1761231415] req@0000000086ccb527 x1846780607005376/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231440 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231415.579688:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607005376/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231440 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231415.579697:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231415.579699:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231415.579702:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005376:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231417.051092:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231420.699266:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607005440:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231420.699388:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231420/real 1761231420] req@0000000088473b3e x1846780607005440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231445 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231420.699439:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607005440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231445 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231420.699454:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231420.699459:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231420.699464:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607005440:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231420.699473:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005504:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231420.699504:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231420/real 1761231420] req@000000004e3f612a x1846780607005504/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231445 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231420.699518:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607005504/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231445 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231420.699526:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231420.699529:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231420.699531:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005504:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231420.699538:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005568:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231420.699559:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231420/real 1761231420] req@00000000daa39bb1 x1846780607005568/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231445 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231420.699571:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607005568/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231445 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231420.699587:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231420.699590:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231420.699593:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005568:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231420.699629:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005632:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231420.699652:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231420/real 1761231420] req@000000003ad149cc x1846780607005632/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231445 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231420.699663:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607005632/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231445 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231420.699673:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231420.699676:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231420.699679:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005632:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231422.171155:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231425.819148:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607005696:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231425.819268:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231425/real 1761231425] req@000000003ad149cc x1846780607005696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231450 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231425.819297:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607005696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231450 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231425.819317:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231425.819346:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231425.819354:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607005696:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231425.819395:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005760:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231425.819447:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231425/real 1761231425] req@00000000daa39bb1 x1846780607005760/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231450 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231425.819462:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607005760/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231450 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231425.819472:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231425.819475:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231425.819479:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005760:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231425.819486:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005824:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231425.819511:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231425/real 1761231425] req@000000004e3f612a x1846780607005824/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231450 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231425.819523:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607005824/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231450 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231425.819532:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231425.819534:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231425.819537:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005824:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231425.819543:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005888:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231425.819564:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231425/real 1761231425] req@0000000088473b3e x1846780607005888/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231450 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231425.819576:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607005888/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231450 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231425.819584:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231425.819587:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231425.819590:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607005888:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231427.291170:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231430.939097:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231430.939111:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231430.939116:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231430.939129:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4303 00000100:00080000:0.0:1761231430.939133:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231430.939142:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231430.939189:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607005952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231430.939234:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231430.939239:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231430.939241:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231430.939278:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4303 00000100:00080000:0.0:1761231430.939281:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231430.939285:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231430.939297:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231430.939314:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607006016/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231430.939329:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231430.939334:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231430.939336:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231430.939341:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4303 00000100:00080000:0.0:1761231430.939344:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231430.939351:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231430.939357:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231430.939377:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607006080/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231430.939392:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231430.939396:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231430.939399:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231430.939405:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4303 00000100:00080000:0.0:1761231430.939408:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231430.939411:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231430.939419:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231430.939431:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607006144/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00010000:00010000:0.0:1761231432.411099:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231436.059055:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607006208:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231436.059138:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231436/real 1761231436] req@0000000086ccb527 x1846780607006208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231461 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231436.059155:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607006208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231461 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231436.059163:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231436.059165:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231436.059180:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607006208:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231436.059201:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607006272:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231436.059221:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231436/real 1761231436] req@000000005a9bdefd x1846780607006272/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231461 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231436.059228:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607006272/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231461 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231436.059233:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231436.059234:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231436.059236:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607006272:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231436.059239:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607006336:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231436.059262:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231436/real 1761231436] req@0000000018a8f5da x1846780607006336/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231461 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231436.059268:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607006336/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231461 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231436.059273:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231436.059275:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231436.059276:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607006336:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231436.059279:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607006400:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231436.059297:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231436/real 1761231436] req@00000000bd684ccc x1846780607006400/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231461 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231436.059303:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607006400/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231461 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231436.059308:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231436.059309:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231436.059311:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607006400:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231437.531188:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231438.555117:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231441.179213:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607006464:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231441.179334:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231441/real 1761231441] req@00000000c7e4e432 x1846780607006464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231466 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231441.179355:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607006464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231466 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231441.179368:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231441.179372:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231441.179376:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607006464:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231441.179386:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607006528:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231441.179412:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231441/real 1761231441] req@00000000f87d0618 x1846780607006528/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231466 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231441.179425:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607006528/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231466 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231441.179434:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231441.179436:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231441.179439:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607006528:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231441.179471:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607006592:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231441.179496:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231441/real 1761231441] req@000000007361665d x1846780607006592/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231466 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231441.179508:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607006592/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231466 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231441.179517:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231441.179519:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231441.179522:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607006592:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231441.179527:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607006656:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231441.179547:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231441/real 1761231441] req@0000000031e043f4 x1846780607006656/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231466 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231441.179558:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607006656/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231466 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231441.179567:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231441.179569:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231441.179572:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607006656:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231442.651168:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231443.675163:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231446.299115:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231446.299129:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231446.299134:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231446.299146:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4318 00000100:00080000:0.0:1761231446.299152:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231446.299160:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231446.299204:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607006720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231446.299242:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231446.299247:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231446.299249:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231446.299253:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4318 00000100:00080000:0.0:1761231446.299256:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231446.299260:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231446.299270:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231446.299282:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607006784/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231446.299293:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231446.299297:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231446.299298:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231446.299301:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4318 00000100:00080000:0.0:1761231446.299303:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231446.299306:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231446.299311:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231446.299321:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607006848/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231446.299331:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231446.299335:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231446.299338:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231446.299342:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4318 00000100:00080000:0.0:1761231446.299344:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231446.299348:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231446.299353:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231446.299370:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607006912/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231446.299707:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231446.299711:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231446.299714:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607006912:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761231446.747073:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231447.771089:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231448.795023:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231449.819068:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231450.843118:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231451.419109:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231451.419121:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231451.419125:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231451.419136:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4323 00000100:00080000:3.0:1761231451.419139:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231451.419148:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231451.419187:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607006976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231451.419224:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231451.419228:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231451.419230:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231451.419234:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4323 00000100:00080000:3.0:1761231451.419236:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231451.419238:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231451.419248:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231451.419260:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607007040/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231451.419270:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231451.419274:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231451.419275:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231451.419278:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4323 00000100:00080000:3.0:1761231451.419281:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231451.419286:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231451.419292:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231451.419311:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607007104/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231451.419314:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607006976:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761231451.419348:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231451.419352:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231451.419353:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231451.419357:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4323 00000100:00080000:3.0:1761231451.419359:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231451.419363:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231451.419366:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231451.419386:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607007168/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231451.419487:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231451/real 1761231451] req@0000000083f16a6b x1846780607006976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231476 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231451.419511:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607006976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231476 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231451.419527:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231451.419531:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231451.419540:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607006976:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231451.419551:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007040:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231451.419587:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231451/real 1761231451] req@00000000a58ca11e x1846780607007040/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231476 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231451.419600:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607007040/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231476 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231451.419608:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231451.419613:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231451.419616:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007040:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231451.419658:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007104:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231451.419684:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231451/real 1761231451] req@000000001f8e688e x1846780607007104/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231476 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231451.419697:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607007104/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231476 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231451.419706:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231451.419709:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231451.419711:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007104:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231451.419718:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007168:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231451.419739:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231451/real 1761231451] req@0000000090b40c66 x1846780607007168/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231476 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231451.419751:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607007168/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231476 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231451.419760:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231451.419762:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231451.419765:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007168:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761231451.867058:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231452.891166:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231453.915162:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231454.939060:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231455.963125:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231456.538938:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231456.538950:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231456.538954:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231456.538968:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4328 00000100:00080000:3.0:1761231456.538971:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231456.538979:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231456.539025:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607007232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231456.539050:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231456.539055:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231456.539056:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231456.539059:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4328 00000100:00080000:3.0:1761231456.539061:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231456.539063:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231456.539072:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231456.539080:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607007296/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231456.539086:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231456.539088:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231456.539089:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231456.539092:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4328 00000100:00080000:3.0:1761231456.539093:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231456.539095:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231456.539097:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231456.539105:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607007360/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231456.539110:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231456.539112:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231456.539113:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231456.539115:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4328 00000100:00080000:3.0:1761231456.539117:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231456.539120:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231456.539123:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231456.539130:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607007424/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231456.539149:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607007232:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231456.539256:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007296:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231456.539274:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007360:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231456.539298:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007424:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761231456.987153:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231458.011095:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231459.035105:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231459.611446:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231456/real 1761231459] req@00000000fc01c41c x1846780607007424/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231481 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231459.611487:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607007424/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231481 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231459.611509:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231459.611516:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231459.611523:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007424:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231459.611534:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231456/real 1761231459] req@00000000408f088c x1846780607007232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231481 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231459.611554:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607007232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231481 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231459.611566:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231459.611570:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231459.611575:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607007232:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231459.611585:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231456/real 1761231459] req@000000002ce934d3 x1846780607007296/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231481 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231459.611615:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607007296/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231481 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231459.611642:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231459.611648:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231459.611654:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007296:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231459.611664:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231456/real 1761231459] req@000000006572b591 x1846780607007360/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231481 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231459.611685:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607007360/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231481 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231459.611699:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231459.611704:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231459.611709:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007360:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761231460.059087:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231461.083127:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231461.659045:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231461.659063:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231461.659069:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231461.659084:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4334 00000100:00080000:0.0:1761231461.659090:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231461.659102:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231461.659150:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607007488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231461.659201:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231461.659209:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231461.659213:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231461.659220:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4334 00000100:00080000:0.0:1761231461.659223:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231461.659230:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231461.659243:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231461.659263:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607007552/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231461.659278:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231461.659283:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231461.659286:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231461.659291:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4334 00000100:00080000:0.0:1761231461.659295:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231461.659300:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231461.659308:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231461.659324:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607007616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231461.659342:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231461.659348:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231461.659351:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761231461.659359:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607007488:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:0.0:1761231461.659365:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4334 00000100:00080000:0.0:1761231461.659373:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231461.659383:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231461.659391:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231461.659413:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607007680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231461.659518:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231461/real 1761231461] req@0000000099514377 x1846780607007488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231486 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231461.659547:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607007488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231486 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231461.659560:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231461.659566:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231461.659574:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607007488:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231461.659588:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007552:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231461.659640:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231461/real 1761231461] req@000000000243251f x1846780607007552/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231486 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231461.659661:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607007552/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231486 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231461.659674:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231461.659678:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231461.659684:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007552:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231461.659726:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007616:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231461.659779:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231461/real 1761231461] req@00000000271a9d8e x1846780607007616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231486 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231461.659878:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607007616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231486 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231461.659894:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231461.659898:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231461.659904:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007616:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231461.659915:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007680:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231461.659969:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231461/real 1761231461] req@0000000023ab3a3f x1846780607007680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231486 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231461.659989:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607007680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231486 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231461.660003:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231461.660007:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231461.660012:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007680:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:3.0:1761231462.107089:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231463.131170:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231464.155173:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231465.179097:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231466.203145:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231466.779052:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231466.779065:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231466.779070:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231466.779082:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4339 00000100:00080000:0.0:1761231466.779087:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231466.779097:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231466.779144:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607007744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231466.779181:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231466.779187:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231466.779191:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231466.779197:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4339 00000100:00080000:0.0:1761231466.779200:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231466.779206:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231466.779218:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231466.779221:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607007744:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231466.779239:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607007808/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231466.779254:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231466.779258:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231466.779260:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231466.779265:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4339 00000100:00080000:0.0:1761231466.779268:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231466.779274:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231466.779279:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231466.779293:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607007872/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231466.779303:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231466.779307:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231466.779310:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231466.779314:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4339 00000100:00080000:0.0:1761231466.779315:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231466.779319:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231466.779323:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231466.779335:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607007936/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231466.779381:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231466/real 1761231466] req@000000005a096598 x1846780607007744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231491 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231466.779404:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607007744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231491 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231466.779416:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231466.779421:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231466.779425:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607007744:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231466.779455:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007808:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231466.779487:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231466/real 1761231466] req@00000000275aeee5 x1846780607007808/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231491 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231466.779500:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607007808/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231491 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231466.779508:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231466.779511:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231466.779514:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007808:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231466.779520:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007872:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231466.779542:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231466/real 1761231466] req@000000002ef1f54f x1846780607007872/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231491 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231466.779554:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607007872/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231491 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231466.779563:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231466.779565:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231466.779567:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007872:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231466.779573:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007936:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231466.779595:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231466/real 1761231466] req@0000000023ab3a3f x1846780607007936/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231491 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231466.779614:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607007936/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231491 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231466.779623:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231466.779625:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231466.779628:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607007936:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:3.0:1761231467.227072:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231468.251127:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231469.275095:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231470.299079:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231471.323122:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231471.899073:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231471.899087:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231471.899092:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231471.899105:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4344 00000100:00080000:3.0:1761231471.899109:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231471.899118:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231471.899164:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607008000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231471.899204:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231471.899209:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231471.899211:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231471.899215:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4344 00000100:00080000:3.0:1761231471.899218:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231471.899222:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231471.899232:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231471.899248:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607008064/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231471.899263:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231471.899268:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231471.899270:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1761231471.899275:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607008000:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761231471.899280:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4344 00000100:00080000:3.0:1761231471.899287:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231471.899294:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231471.899299:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231471.899312:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607008128/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231471.899322:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231471.899326:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231471.899328:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231471.899332:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4344 00000100:00080000:3.0:1761231471.899334:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231471.899338:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231471.899343:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231471.899354:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607008192/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231471.899405:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231471/real 1761231471] req@000000006572b591 x1846780607008000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231496 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231471.899428:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607008000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231496 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231471.899441:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231471.899445:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231471.899450:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607008000:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231471.899479:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008064:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231471.899510:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231471/real 1761231471] req@000000002ce934d3 x1846780607008064/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231496 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231471.899523:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607008064/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231496 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231471.899531:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231471.899533:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231471.899536:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008064:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231471.899542:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008128:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231471.899564:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231471/real 1761231471] req@00000000408f088c x1846780607008128/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231496 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231471.899575:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607008128/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231496 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231471.899584:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231471.899586:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231471.899589:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008128:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231471.899595:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008192:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231471.899616:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231471/real 1761231471] req@00000000fc01c41c x1846780607008192/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231496 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231471.899627:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607008192/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231496 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231471.899680:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231471.899684:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231471.899688:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008192:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231472.347180:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231473.371126:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231474.395128:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231475.419117:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231476.443135:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231477.018920:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231477.018934:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231477.018939:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231477.018951:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4349 00000100:00080000:3.0:1761231477.018955:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231477.018963:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231477.019007:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607008256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231477.019038:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231477.019044:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231477.019046:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231477.019052:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4349 00000100:00080000:3.0:1761231477.019055:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231477.019059:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231477.019070:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231477.019082:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607008320/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231477.019092:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231477.019096:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231477.019098:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231477.019101:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4349 00000100:00080000:3.0:1761231477.019103:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231477.019106:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231477.019110:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231477.019122:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607008384/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231477.019131:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231477.019135:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231477.019136:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231477.019141:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4349 00000100:00080000:3.0:1761231477.019144:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231477.019148:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231477.019153:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231477.019162:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607008256:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231477.019167:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607008448/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231477.019274:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231477/real 1761231477] req@0000000090b40c66 x1846780607008256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231502 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231477.019296:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607008256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231502 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231477.019308:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231477.019312:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231477.019317:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607008256:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231477.019325:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008320:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231477.019351:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231477/real 1761231477] req@000000001f8e688e x1846780607008320/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231502 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231477.019363:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607008320/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231502 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231477.019371:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231477.019373:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231477.019376:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008320:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231477.019382:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008384:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231477.019405:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231477/real 1761231477] req@00000000a58ca11e x1846780607008384/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231502 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231477.019417:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607008384/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231502 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231477.019426:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231477.019428:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231477.019431:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008384:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231477.019462:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008448:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231477.019485:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231477/real 1761231477] req@0000000083f16a6b x1846780607008448/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231502 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231477.019496:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607008448/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231502 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231477.019505:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231477.019507:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231477.019510:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008448:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231477.467126:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231478.491177:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231479.515151:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231480.539061:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231481.563143:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231482.139062:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231482.139075:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231482.139081:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231482.139093:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4354 00000100:00080000:1.0:1761231482.139097:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231482.139105:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231482.139149:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607008512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231482.139185:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231482.139190:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231482.139192:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231482.139197:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4354 00000100:00080000:1.0:1761231482.139200:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231482.139203:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231482.139213:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231482.139228:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607008576/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231482.139240:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231482.139245:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231482.139247:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231482.139251:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4354 00000100:00080000:1.0:1761231482.139254:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1761231482.139260:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607008512:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:1.0:1761231482.139264:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231482.139277:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231482.139291:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607008640/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231482.139302:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231482.139306:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231482.139307:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231482.139312:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4354 00000100:00080000:1.0:1761231482.139315:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231482.139318:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231482.139323:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231482.139334:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607008704/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231482.139385:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231482/real 1761231482] req@00000000db10af3c x1846780607008512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231507 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231482.139406:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607008512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231507 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231482.139420:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231482.139423:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231482.139428:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607008512:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231482.139436:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008576:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231482.139463:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231482/real 1761231482] req@00000000d07717c4 x1846780607008576/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231507 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231482.139476:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607008576/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231507 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231482.139514:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231482.139517:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231482.139521:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008576:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231482.139551:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008640:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231482.139581:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231482/real 1761231482] req@00000000b72feabb x1846780607008640/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231507 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231482.139594:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607008640/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231507 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231482.139603:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231482.139605:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231482.139608:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008640:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231482.139614:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008704:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231482.139637:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231482/real 1761231482] req@00000000659d78e3 x1846780607008704/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231507 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231482.139648:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607008704/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231507 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231482.139657:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231482.139659:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231482.139662:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008704:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231482.587077:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231483.611140:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231484.635063:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231485.659090:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231486.683121:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231487.259004:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231487.259018:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231487.259022:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231487.259035:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4359 00000100:00080000:1.0:1761231487.259039:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231487.259048:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231487.259093:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607008768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231487.259128:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231487.259133:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231487.259135:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231487.259140:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4359 00000100:00080000:1.0:1761231487.259142:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231487.259146:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231487.259155:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231487.259168:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607008832/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231487.259178:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231487.259181:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231487.259183:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231487.259187:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4359 00000100:00080000:1.0:1761231487.259190:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231487.259193:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231487.259197:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231487.259206:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607008896/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231487.259215:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231487.259219:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231487.259221:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231487.259225:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4359 00000100:00080000:1.0:1761231487.259228:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231487.259233:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231487.259238:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231487.259248:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607008960/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231487.259266:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607008768:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231487.259376:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231487/real 1761231487] req@00000000659d78e3 x1846780607008768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231512 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231487.259397:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607008768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231512 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231487.259457:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231487.259462:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231487.259470:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607008768:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231487.259481:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008832:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231487.259518:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231487/real 1761231487] req@00000000b72feabb x1846780607008832/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231512 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231487.259532:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607008832/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231512 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231487.259542:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231487.259545:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231487.259549:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008832:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231487.259555:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008896:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231487.259578:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231487/real 1761231487] req@00000000d07717c4 x1846780607008896/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231512 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231487.259589:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607008896/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231512 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231487.259598:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231487.259601:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231487.259603:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008896:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231487.259635:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008960:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231487.259659:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231487/real 1761231487] req@00000000db10af3c x1846780607008960/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231512 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231487.259671:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607008960/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231512 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231487.259680:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231487.259682:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231487.259684:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607008960:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231487.707170:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231488.731125:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231489.755140:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231490.779053:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231491.803130:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231492.378934:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231492.378950:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231492.378955:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231492.378972:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4364 00000100:00080000:1.0:1761231492.378978:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231492.378989:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231492.379044:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607009024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231492.379087:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231492.379092:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231492.379094:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231492.379100:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4364 00000100:00080000:1.0:1761231492.379103:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231492.379108:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231492.379119:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231492.379135:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607009088/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231492.379146:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231492.379150:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231492.379152:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231492.379157:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4364 00000100:00080000:1.0:1761231492.379174:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231492.379179:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231492.379186:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231492.379198:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607009152/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231492.379211:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607009024:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:1.0:1761231492.379212:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231492.379231:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231492.379234:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231492.379240:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4364 00000100:00080000:1.0:1761231492.379243:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231492.379248:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231492.379253:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231492.379269:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607009216/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231492.379360:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231492/real 1761231492] req@00000000db10af3c x1846780607009024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231517 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231492.379383:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607009024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231517 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231492.379396:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231492.379399:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231492.379404:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607009024:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231492.379413:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009088:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231492.379443:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231492/real 1761231492] req@00000000d07717c4 x1846780607009088/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231517 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231492.379456:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607009088/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231517 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231492.379465:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231492.379467:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231492.379471:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009088:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231492.379499:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009152:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231492.379523:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231492/real 1761231492] req@00000000b72feabb x1846780607009152/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231517 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231492.379534:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607009152/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231517 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231492.379544:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231492.379547:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231492.379550:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009152:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231492.379555:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009216:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231492.379575:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231492/real 1761231492] req@00000000659d78e3 x1846780607009216/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231517 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231492.379586:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607009216/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231517 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231492.379595:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231492.379598:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231492.379601:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009216:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231492.827118:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231493.851139:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231494.875135:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231495.899110:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231496.923088:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231497.499042:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231497.499055:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231497.499060:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231497.499073:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4369 00000100:00080000:3.0:1761231497.499077:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231497.499086:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231497.499133:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607009280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231497.499171:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231497.499176:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231497.499178:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231497.499182:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4369 00000100:00080000:3.0:1761231497.499185:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231497.499189:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231497.499198:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231497.499212:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607009344/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231497.499222:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231497.499226:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231497.499228:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231497.499232:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4369 00000100:00080000:3.0:1761231497.499233:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231497.499237:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231497.499241:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231497.499283:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607009408/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231497.499294:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231497.499298:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231497.499300:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231497.499303:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4369 00000100:00080000:3.0:1761231497.499306:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231497.499310:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231497.499314:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231497.499326:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607009472/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231497.499335:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607009280:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231497.499449:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231497/real 1761231497] req@0000000031e043f4 x1846780607009280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231522 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231497.499471:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607009280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231522 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231497.499484:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231497.499487:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231497.499492:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607009280:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231497.499501:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009344:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231497.499526:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231497/real 1761231497] req@000000007361665d x1846780607009344/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231522 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231497.499539:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607009344/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231522 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231497.499547:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231497.499550:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231497.499552:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009344:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231497.499558:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009408:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231497.499579:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231497/real 1761231497] req@00000000f87d0618 x1846780607009408/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231522 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231497.499590:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607009408/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231522 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231497.499599:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231497.499601:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231497.499604:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009408:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231497.499637:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009472:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231497.499659:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231497/real 1761231497] req@00000000c7e4e432 x1846780607009472/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231522 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231497.499670:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607009472/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231522 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231497.499680:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231497.499683:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231497.499686:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009472:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231497.947147:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231498.971034:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231499.995164:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231501.019130:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231502.043112:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231502.619052:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231502.619065:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231502.619071:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231502.619084:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4375 00000100:00080000:2.0:1761231502.619089:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231502.619098:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231502.619140:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607009536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231502.619253:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231502.619261:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231502.619264:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1761231502.619271:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607009536:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:2.0:1761231502.619271:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4375 00000100:00080000:2.0:1761231502.619275:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231502.619281:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231502.619295:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231502.619318:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607009600/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231502.619334:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231502.619344:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231502.619346:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231502.619352:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4375 00000100:00080000:2.0:1761231502.619355:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231502.619360:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231502.619368:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231502.619385:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607009664/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231502.619394:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231502/real 1761231502] req@0000000064978143 x1846780607009536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231527 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:2.0:1761231502.619400:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231502.619407:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231502.619409:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231502.619414:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4375 00000100:00080000:2.0:1761231502.619418:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1761231502.619421:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607009536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231527 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:2.0:1761231502.619423:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231502.619430:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00080000:0.0:1761231502.619439:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231502.619444:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761231502.619444:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607009728/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231502.619451:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607009536:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231502.619490:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009600:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231502.619545:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231502/real 1761231502] req@0000000091106a97 x1846780607009600/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231527 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231502.619564:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607009600/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231527 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231502.619574:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231502.619576:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231502.619581:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009600:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231502.619588:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009664:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231502.619618:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231502/real 1761231502] req@0000000094135f44 x1846780607009664/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231527 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231502.619630:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607009664/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231527 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231502.619639:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231502.619642:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231502.619644:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009664:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231502.619650:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009728:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231502.619672:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231502/real 1761231502] req@00000000c069b612 x1846780607009728/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231527 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231502.619683:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607009728/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231527 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231502.619692:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231502.619694:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231502.619697:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009728:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231503.067154:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231504.091198:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231505.115203:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231506.139099:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231507.163173:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231507.738965:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231507.738978:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231507.738983:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231507.738996:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4380 00000100:00080000:2.0:1761231507.739001:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231507.739009:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231507.739053:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607009792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231507.739086:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231507.739090:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231507.739092:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231507.739125:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4380 00000100:00080000:2.0:1761231507.739128:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231507.739132:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231507.739142:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231507.739155:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607009856/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231507.739166:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231507.739171:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231507.739173:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231507.739179:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4380 00000100:00080000:2.0:1761231507.739182:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231507.739186:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231507.739191:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231507.739203:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607009920/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231507.739215:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761231507.739217:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607009792:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:2.0:1761231507.739220:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231507.739225:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231507.739235:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4380 00000100:00080000:2.0:1761231507.739241:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231507.739247:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231507.739257:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231507.739281:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607009984/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231507.739340:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231507/real 1761231507] req@0000000017386078 x1846780607009792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231532 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231507.739362:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607009792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231532 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231507.739374:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231507.739378:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231507.739382:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607009792:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231507.739391:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009856:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231507.739418:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231507/real 1761231507] req@00000000f38f27df x1846780607009856/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231532 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231507.739430:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607009856/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231532 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231507.739438:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231507.739440:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231507.739443:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009856:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231507.739469:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009920:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231507.739492:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231507/real 1761231507] req@00000000c069b612 x1846780607009920/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231532 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231507.739504:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607009920/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231532 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231507.739513:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231507.739517:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231507.739519:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009920:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231507.739525:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009984:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231507.739546:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231507/real 1761231507] req@0000000094135f44 x1846780607009984/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231532 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231507.739556:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607009984/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231532 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231507.739565:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231507.739568:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231507.739570:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607009984:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231508.187152:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231509.211149:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231510.235147:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231511.259148:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231512.283143:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231512.858966:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231512.858980:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231512.858985:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231512.858999:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4385 00000100:00080000:2.0:1761231512.859004:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231512.859012:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231512.859098:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607010048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231512.859135:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231512.859140:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231512.859142:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231512.859147:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4385 00000100:00080000:2.0:1761231512.859150:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231512.859154:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231512.859163:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231512.859181:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607010112/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231512.859192:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231512.859200:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231512.859202:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231512.859207:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4385 00000100:00080000:2.0:1761231512.859210:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231512.859213:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231512.859219:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231512.859232:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607010048:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:2.0:1761231512.859233:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607010176/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231512.859262:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231512.859266:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231512.859268:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231512.859275:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4385 00000100:00080000:2.0:1761231512.859277:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231512.859280:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231512.859285:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231512.859296:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607010240/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231512.859376:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231512/real 1761231512] req@0000000091106a97 x1846780607010048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231537 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231512.859392:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607010048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231537 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231512.859401:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231512.859405:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231512.859408:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607010048:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231512.859415:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010112:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231512.859433:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231512/real 1761231512] req@0000000064978143 x1846780607010112/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231537 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231512.859441:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607010112/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231537 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231512.859446:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231512.859447:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231512.859449:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010112:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231512.859476:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010176:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231512.859491:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231512/real 1761231512] req@0000000094135f44 x1846780607010176/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231537 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231512.859497:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607010176/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231537 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231512.859503:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231512.859504:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231512.859505:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010176:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231512.859509:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010240:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231512.859520:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231512/real 1761231512] req@00000000c069b612 x1846780607010240/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231537 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231512.859527:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607010240/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231537 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231512.859532:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231512.859534:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231512.859536:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010240:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231513.307112:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231514.331121:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231515.355034:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231516.379174:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231517.403129:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231517.979004:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231517.979018:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231517.979023:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231517.979035:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4390 00000100:00080000:2.0:1761231517.979040:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231517.979049:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231517.979094:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607010304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231517.979129:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231517.979133:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231517.979135:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231517.979140:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4390 00000100:00080000:2.0:1761231517.979143:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231517.979146:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231517.979155:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231517.979170:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607010368/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231517.979180:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231517.979184:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231517.979185:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231517.979189:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4390 00000100:00080000:2.0:1761231517.979191:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231517.979194:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231517.979198:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231517.979207:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607010432/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231517.979217:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231517.979221:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231517.979223:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231517.979227:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4390 00000100:00080000:2.0:1761231517.979231:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231517.979235:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231517.979240:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231517.979249:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607010496/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231517.979264:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607010304:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231517.979374:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231517/real 1761231517] req@00000000f38f27df x1846780607010304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231542 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231517.979395:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607010304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231542 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231517.979408:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231517.979411:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231517.979416:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607010304:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231517.979424:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010368:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231517.979450:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231517/real 1761231517] req@0000000017386078 x1846780607010368/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231542 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231517.979462:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607010368/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231542 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231517.979470:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231517.979472:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231517.979475:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010368:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231517.979481:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010432:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231517.979503:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231517/real 1761231517] req@00000000c069b612 x1846780607010432/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231542 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231517.979515:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607010432/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231542 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231517.979524:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231517.979526:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231517.979529:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010432:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231517.979557:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010496:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231517.979580:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231517/real 1761231517] req@0000000094135f44 x1846780607010496/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231542 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231517.979591:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607010496/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231542 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231517.979599:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231517.979601:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231517.979604:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010496:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231518.427172:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231519.451150:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231520.475178:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231521.499170:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231522.523008:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231523.099045:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231523.099058:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231523.099062:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231523.099074:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4395 00000100:00080000:1.0:1761231523.099079:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231523.099087:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231523.099134:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607010560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231523.099170:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231523.099176:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231523.099178:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231523.099182:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4395 00000100:00080000:1.0:1761231523.099185:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231523.099188:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231523.099198:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231523.099211:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607010624/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231523.099222:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231523.099227:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231523.099229:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231523.099234:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4395 00000100:00080000:1.0:1761231523.099236:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231523.099240:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231523.099244:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231523.099255:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607010560:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761231523.099260:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607010688/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231523.099276:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231523.099280:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231523.099281:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231523.099285:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4395 00000100:00080000:1.0:1761231523.099287:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231523.099291:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231523.099295:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231523.099311:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607010752/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231523.099381:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010624:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231523.099419:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010688:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231523.099439:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010752:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231523.547118:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231524.570997:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231525.595136:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231526.171182:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231523/real 1761231526] req@00000000d07717c4 x1846780607010688/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231548 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231526.171203:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607010688/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231548 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231526.171212:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231526.171215:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231526.171218:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010688:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231526.171222:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231523/real 1761231526] req@00000000db10af3c x1846780607010752/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231548 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231526.171228:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607010752/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231548 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231526.171232:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231526.171234:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231526.171236:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010752:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231526.171239:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231523/real 1761231526] req@00000000659d78e3 x1846780607010560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231548 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231526.171249:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607010560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231548 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231526.171261:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231526.171264:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231526.171266:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607010560:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231526.171271:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231523/real 1761231526] req@00000000b72feabb x1846780607010624/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231548 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231526.171280:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607010624/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231548 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231526.171286:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231526.171288:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231526.171291:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010624:192.168.201.104@tcp:38:kworker/u8:2.0 00010000:00010000:0.0:1761231526.619101:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231527.642993:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231528.218967:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231528.218976:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231528.218978:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231528.218987:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4400 00000100:00080000:2.0:1761231528.218989:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231528.218994:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231528.219021:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607010816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231528.219041:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231528.219044:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231528.219045:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231528.219048:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4400 00000100:00080000:2.0:1761231528.219049:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231528.219051:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231528.219057:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231528.219064:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607010880/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231528.219069:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231528.219072:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231528.219073:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231528.219075:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4400 00000100:00080000:2.0:1761231528.219077:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231528.219078:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231528.219081:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231528.219086:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607010944/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231528.219090:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231528.219092:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231528.219092:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231528.219094:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4400 00000100:00080000:2.0:1761231528.219095:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231528.219097:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231528.219101:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231528.219107:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607011008/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231528.219153:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607010816:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231528.219281:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231528/real 1761231528] req@0000000064978143 x1846780607010816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231553 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231528.219302:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607010816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231553 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231528.219312:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231528.219315:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231528.219319:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607010816:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231528.219328:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010880:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231528.219358:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231528/real 1761231528] req@0000000091106a97 x1846780607010880/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231553 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231528.219369:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607010880/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231553 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231528.219377:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231528.219379:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231528.219382:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010880:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231528.219388:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010944:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231528.219414:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231528/real 1761231528] req@0000000094135f44 x1846780607010944/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231553 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231528.219424:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607010944/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231553 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231528.219433:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231528.219436:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231528.219439:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607010944:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231528.219445:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011008:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231528.219469:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231528/real 1761231528] req@00000000c069b612 x1846780607011008/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231553 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231528.219480:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607011008/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231553 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231528.219489:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231528.219491:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231528.219495:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011008:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231528.667009:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231529.691094:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231530.715020:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231531.739031:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231532.763026:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231533.338982:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231533.338991:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231533.338994:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231533.339003:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4405 00000100:00080000:1.0:1761231533.339006:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231533.339012:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231533.339046:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607011072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231533.339068:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231533.339071:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231533.339072:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231533.339074:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4405 00000100:00080000:1.0:1761231533.339076:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231533.339078:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231533.339084:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231533.339090:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607011136/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231533.339095:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231533.339098:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231533.339099:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231533.339101:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4405 00000100:00080000:1.0:1761231533.339102:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231533.339103:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231533.339107:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231533.339112:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607011200/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231533.339117:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231533.339119:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231533.339119:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231533.339121:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4405 00000100:00080000:1.0:1761231533.339122:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231533.339124:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231533.339127:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231533.339132:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607011264/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231533.339173:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607011072:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231533.339282:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231533/real 1761231533] req@00000000b72feabb x1846780607011072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231558 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231533.339297:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607011072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231558 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231533.339305:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231533.339309:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231533.339312:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607011072:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231533.339316:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011136:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231533.339332:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231533/real 1761231533] req@00000000659d78e3 x1846780607011136/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231558 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231533.339340:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607011136/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231558 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231533.339344:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231533.339346:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231533.339347:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011136:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231533.339350:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011200:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231533.339365:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231533/real 1761231533] req@00000000db10af3c x1846780607011200/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231558 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231533.339372:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607011200/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231558 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231533.339377:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231533.339378:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231533.339380:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011200:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231533.339383:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011264:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231533.339394:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231533/real 1761231533] req@00000000d07717c4 x1846780607011264/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231558 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231533.339401:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607011264/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231558 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231533.339406:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231533.339407:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231533.339409:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011264:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231533.787031:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231534.810997:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231535.835023:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231536.859006:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231537.882983:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231538.458990:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231538.458998:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231538.459001:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231538.459010:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4410 00000100:00080000:3.0:1761231538.459014:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231538.459020:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231538.459051:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607011328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231538.459074:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231538.459077:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231538.459078:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231538.459081:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4410 00000100:00080000:3.0:1761231538.459082:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231538.459084:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231538.459091:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231538.459098:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607011392/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231538.459104:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231538.459105:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231538.459106:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231538.459108:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4410 00000100:00080000:3.0:1761231538.459109:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231538.459111:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231538.459113:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231538.459120:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607011456/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231538.459124:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231538.459127:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231538.459129:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231538.459131:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4410 00000100:00080000:3.0:1761231538.459133:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231538.459138:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231538.459141:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231538.459147:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607011520/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231538.459172:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607011328:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231538.459282:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231538/real 1761231538] req@00000000bd684ccc x1846780607011328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231563 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231538.459301:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607011328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231563 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231538.459310:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231538.459313:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231538.459316:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607011328:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231538.459320:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011392:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231538.459336:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231538/real 1761231538] req@0000000018a8f5da x1846780607011392/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231563 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231538.459342:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607011392/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231563 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231538.459346:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231538.459348:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231538.459349:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011392:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231538.459353:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011456:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231538.459365:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231538/real 1761231538] req@000000005a9bdefd x1846780607011456/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231563 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231538.459372:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607011456/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231563 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231538.459377:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231538.459378:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231538.459380:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011456:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231538.459384:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011520:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231538.459394:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231538/real 1761231538] req@0000000086ccb527 x1846780607011520/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231563 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231538.459400:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607011520/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231563 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231538.459405:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231538.459406:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231538.459417:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011520:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231538.907009:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231539.931041:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231540.954992:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231541.979030:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231543.003028:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231543.578906:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231543.578914:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231543.578917:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231543.578926:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4415 00000100:00080000:3.0:1761231543.578929:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231543.578934:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231543.578964:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607011584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231543.578985:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231543.578987:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231543.578988:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231543.578991:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4415 00000100:00080000:3.0:1761231543.578992:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231543.578994:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231543.579000:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231543.579007:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607011648/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231543.579016:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231543.579018:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231543.579020:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231543.579022:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4415 00000100:00080000:3.0:1761231543.579023:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231543.579024:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231543.579026:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231543.579033:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607011712/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231543.579038:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231543.579040:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231543.579041:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231543.579044:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4415 00000100:00080000:3.0:1761231543.579045:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231543.579046:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231543.579049:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231543.579055:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607011776/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231543.579061:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607011584:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231543.579137:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231543/real 1761231543] req@0000000088473b3e x1846780607011584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231568 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231543.579149:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607011584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231568 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231543.579156:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231543.579158:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231543.579161:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607011584:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231543.579165:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011648:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231543.579180:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231543/real 1761231543] req@000000004e3f612a x1846780607011648/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231568 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231543.579187:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607011648/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231568 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231543.579191:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231543.579193:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231543.579194:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011648:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231543.579198:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011712:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231543.579209:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231543/real 1761231543] req@00000000daa39bb1 x1846780607011712/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231568 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231543.579215:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607011712/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231568 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231543.579220:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231543.579221:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231543.579223:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011712:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231543.579243:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011776:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231543.579254:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231543/real 1761231543] req@000000003ad149cc x1846780607011776/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231568 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231543.579260:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607011776/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231568 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231543.579265:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231543.579267:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231543.579268:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011776:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231544.027182:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231545.051121:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231546.075097:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231547.099119:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231548.123206:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231548.699086:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231548.699101:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231548.699105:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231548.699118:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4421 00000100:00080000:0.0:1761231548.699123:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231548.699131:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231548.699176:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607011840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231548.699217:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231548.699221:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231548.699223:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231548.699228:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4421 00000100:00080000:0.0:1761231548.699230:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231548.699234:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231548.699245:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231548.699258:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607011904/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231548.699272:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231548.699276:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231548.699279:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761231548.699289:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607011840:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:0.0:1761231548.699290:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4421 00000100:00080000:0.0:1761231548.699297:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231548.699305:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231548.699312:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231548.699325:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607011968/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231548.699335:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231548.699339:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231548.699342:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231548.699346:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4421 00000100:00080000:0.0:1761231548.699348:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231548.699351:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231548.699357:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231548.699367:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607012032/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231548.699417:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231548/real 1761231548] req@00000000271a9d8e x1846780607011840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231573 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231548.699440:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607011840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231573 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231548.699455:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231548.699458:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231548.699463:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607011840:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231548.699516:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011904:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231548.699551:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231548/real 1761231548] req@000000000243251f x1846780607011904/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231573 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231548.699564:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607011904/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231573 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231548.699573:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231548.699576:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231548.699580:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011904:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231548.699586:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011968:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231548.699608:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231548/real 1761231548] req@0000000099514377 x1846780607011968/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231573 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231548.699620:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607011968/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231573 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231548.699630:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231548.699632:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231548.699635:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607011968:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231548.699641:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012032:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231548.699665:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231548/real 1761231548] req@00000000b8f2be78 x1846780607012032/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231573 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231548.699678:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607012032/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231573 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231548.699688:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231548.699692:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231548.699696:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012032:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231549.147091:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231550.171158:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231551.195148:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231552.219163:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231553.243128:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231553.819120:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231553.819134:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231553.819139:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231553.819153:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4426 00000100:00080000:1.0:1761231553.819157:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231553.819166:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231553.819215:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607012096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231553.819257:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231553.819262:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231553.819263:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231553.819268:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4426 00000100:00080000:1.0:1761231553.819272:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231553.819276:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231553.819287:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231553.819299:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607012160/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231553.819310:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231553.819313:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231553.819315:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231553.819318:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4426 00000100:00080000:1.0:1761231553.819320:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231553.819323:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231553.819328:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231553.819342:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607012224/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231553.819351:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231553.819354:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231553.819356:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231553.819360:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4426 00000100:00080000:1.0:1761231553.819362:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231553.819366:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231553.819371:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231553.819381:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607012288/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231553.819399:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607012096:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231553.819594:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231553/real 1761231553] req@00000000d07717c4 x1846780607012096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231578 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231553.819619:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607012096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231578 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231553.819634:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231553.819639:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231553.819644:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607012096:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231553.819653:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012160:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231553.819682:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231553/real 1761231553] req@00000000db10af3c x1846780607012160/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231578 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231553.819695:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607012160/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231578 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231553.819711:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231553.819714:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231553.819718:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012160:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231553.819724:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012224:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231553.819746:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231553/real 1761231553] req@00000000659d78e3 x1846780607012224/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231578 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231553.819757:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607012224/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231578 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231553.819766:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231553.819768:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231553.819771:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012224:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231553.819812:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012288:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231553.819884:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231553/real 1761231553] req@00000000b72feabb x1846780607012288/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231578 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231553.819899:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607012288/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231578 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231553.819909:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231553.819913:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231553.819916:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012288:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231554.267216:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231555.291201:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231556.315223:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231557.339127:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231558.363174:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231558.938925:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231558.938938:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231558.938942:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231558.938955:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4431 00000100:00080000:3.0:1761231558.938959:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231558.938967:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231558.939008:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607012352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231558.939042:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231558.939047:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231558.939049:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231558.939053:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4431 00000100:00080000:3.0:1761231558.939056:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231558.939059:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231558.939068:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231558.939084:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607012416/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231558.939096:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231558.939101:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231558.939103:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231558.939108:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4431 00000100:00080000:3.0:1761231558.939111:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231558.939115:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1761231558.939123:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607012352:192.168.201.104@tcp:250:kworker/u8:0.0 00000008:00100000:3.0:1761231558.939123:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231558.939142:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607012480/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231558.939156:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231558.939160:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231558.939162:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231558.939166:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4431 00000100:00080000:3.0:1761231558.939168:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231558.939172:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231558.939177:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231558.939190:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607012544/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231558.939242:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231558/real 1761231558] req@000000003ad149cc x1846780607012352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231583 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231558.939265:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607012352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231583 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231558.939278:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231558.939282:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231558.939287:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607012352:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231558.939295:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012416:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231558.939325:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231558/real 1761231558] req@00000000daa39bb1 x1846780607012416/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231583 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231558.939377:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607012416/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231583 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231558.939393:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231558.939398:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231558.939403:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012416:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231558.939445:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012480:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231558.939492:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231558/real 1761231558] req@000000004e3f612a x1846780607012480/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231583 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231558.939506:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607012480/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231583 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231558.939516:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231558.939518:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231558.939522:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012480:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231558.939529:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012544:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231558.939552:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231558/real 1761231558] req@0000000088473b3e x1846780607012544/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231583 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231558.939564:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607012544/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231583 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231558.939572:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231558.939574:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231558.939577:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012544:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231559.387102:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231560.411145:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231561.435147:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231562.459095:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231563.483107:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231564.058980:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231564.058997:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231564.059002:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231564.059018:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4436 00000100:00080000:3.0:1761231564.059024:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231564.059035:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231564.059086:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607012608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231564.059124:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231564.059129:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231564.059131:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231564.059136:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4436 00000100:00080000:3.0:1761231564.059139:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231564.059144:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231564.059154:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231564.059167:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607012672/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231564.059177:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231564.059180:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231564.059182:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231564.059186:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4436 00000100:00080000:3.0:1761231564.059188:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231564.059192:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231564.059197:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231564.059209:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607012736/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231564.059218:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231564.059222:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231564.059223:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231564.059227:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4436 00000100:00080000:3.0:1761231564.059230:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231564.059234:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231564.059240:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231564.059244:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607012608:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231564.059284:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607012800/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231564.059428:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231564/real 1761231564] req@0000000086ccb527 x1846780607012608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231589 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231564.059459:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607012608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231589 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231564.059478:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231564.059483:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231564.059490:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607012608:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231564.059502:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012672:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231564.059553:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231564/real 1761231564] req@000000005a9bdefd x1846780607012672/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231589 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231564.059576:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607012672/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231589 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231564.059590:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231564.059594:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231564.059600:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012672:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231564.059610:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012736:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231564.059654:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231564/real 1761231564] req@0000000018a8f5da x1846780607012736/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231589 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231564.059674:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607012736/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231589 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231564.059689:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231564.059693:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231564.059698:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012736:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231564.059741:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012800:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231564.059789:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231564/real 1761231564] req@00000000bd684ccc x1846780607012800/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231589 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231564.059809:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607012800/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231589 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231564.059824:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231564.059884:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231564.059892:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012800:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231564.507090:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231565.530993:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231566.555067:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231567.579105:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761231568.603044:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231569.179062:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231569.179076:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231569.179081:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231569.179094:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4441 00000100:00080000:0.0:1761231569.179099:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231569.179108:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231569.179153:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607012864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231569.179218:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231569.179224:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231569.179226:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231569.179231:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4441 00000100:00080000:0.0:1761231569.179234:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231569.179238:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231569.179249:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231569.179263:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607012928/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231569.179274:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231569.179278:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231569.179280:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231569.179285:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4441 00000100:00080000:0.0:1761231569.179288:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231569.179292:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231569.179297:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231569.179310:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607012992/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231569.179319:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231569.179323:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231569.179325:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231569.179329:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4441 00000100:00080000:0.0:1761231569.179332:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231569.179337:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231569.179344:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231569.179344:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607012864:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231569.179364:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607013056/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231569.179473:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231569/real 1761231569] req@00000000fcce404c x1846780607012864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231594 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231569.179495:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607012864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231594 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231569.179507:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231569.179511:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231569.179516:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607012864:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231569.179524:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012928:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231569.179551:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231569/real 1761231569] req@000000007f60e210 x1846780607012928/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231594 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231569.179564:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607012928/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231594 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231569.179572:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231569.179575:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231569.179577:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012928:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231569.179584:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012992:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231569.179607:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231569/real 1761231569] req@0000000001b2f813 x1846780607012992/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231594 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231569.179618:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607012992/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231594 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231569.179627:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231569.179631:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231569.179634:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607012992:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231569.179665:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013056:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231569.179688:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231569/real 1761231569] req@00000000b8f2be78 x1846780607013056/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231594 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231569.179700:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607013056/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231594 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231569.179709:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231569.179712:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231569.179715:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013056:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231569.627319:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231570.651151:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231571.675067:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231572.699083:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231573.723137:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231574.298974:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231574.298986:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231574.298989:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231574.298999:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4446 00000100:00080000:3.0:1761231574.299002:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231574.299008:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231574.299044:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607013120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231574.299073:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231574.299077:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231574.299078:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231574.299082:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4446 00000100:00080000:3.0:1761231574.299085:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231574.299091:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231574.299113:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231574.299121:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607013120:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231574.299128:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607013184/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231574.299140:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231574.299144:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231574.299146:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231574.299149:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4446 00000100:00080000:3.0:1761231574.299151:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231574.299155:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231574.299160:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231574.299170:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607013248/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231574.299178:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231574.299181:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231574.299183:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231574.299187:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4446 00000100:00080000:3.0:1761231574.299189:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231574.299192:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231574.299196:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231574.299206:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607013312/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231574.299231:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231574/real 1761231574] req@00000000c7e4e432 x1846780607013120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231599 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231574.299245:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607013120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231599 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231574.299256:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231574.299259:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231574.299261:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607013120:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231574.299279:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013184:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231574.299294:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231574/real 1761231574] req@00000000f87d0618 x1846780607013184/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231599 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231574.299301:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607013184/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231599 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231574.299305:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231574.299307:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231574.299308:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013184:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231574.299311:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013248:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231574.299322:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231574/real 1761231574] req@000000007361665d x1846780607013248/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231599 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231574.299328:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607013248/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231599 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231574.299333:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231574.299334:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231574.299335:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013248:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231574.299338:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013312:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231574.299350:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231574/real 1761231574] req@0000000031e043f4 x1846780607013312/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231599 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231574.299355:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607013312/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231599 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231574.299360:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231574.299361:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231574.299363:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013312:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231574.747159:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231575.771198:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231576.795152:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231577.819206:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231578.843193:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231579.418966:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231579.418979:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231579.418984:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231579.418996:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4451 00000100:00080000:3.0:1761231579.419000:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231579.419007:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231579.419101:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607013376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231579.419131:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231579.419136:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231579.419138:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231579.419143:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4451 00000100:00080000:3.0:1761231579.419145:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231579.419149:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231579.419158:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231579.419179:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607013440/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231579.419187:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231579.419189:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231579.419190:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231579.419193:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4451 00000100:00080000:3.0:1761231579.419195:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231579.419198:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231579.419202:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231579.419213:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607013504/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231579.419220:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231579.419223:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231579.419224:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231579.419229:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4451 00000100:00100000:0.0:1761231579.419231:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607013376:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761231579.419232:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231579.419238:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231579.419242:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231579.419250:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607013568/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231579.419317:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231579/real 1761231579] req@0000000083f16a6b x1846780607013376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231604 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231579.419330:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607013376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231604 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231579.419338:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231579.419340:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231579.419343:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607013376:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231579.419348:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013440:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231579.419364:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231579/real 1761231579] req@00000000a58ca11e x1846780607013440/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231604 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231579.419376:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607013440/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231604 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231579.419381:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231579.419382:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231579.419384:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013440:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231579.419404:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013504:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231579.419419:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231579/real 1761231579] req@000000001f8e688e x1846780607013504/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231604 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231579.419425:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607013504/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231604 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231579.419431:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231579.419432:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231579.419434:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013504:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231579.419437:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013568:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231579.419449:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231579/real 1761231579] req@0000000090b40c66 x1846780607013568/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231604 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231579.419455:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607013568/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231604 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231579.419461:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231579.419462:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231579.419464:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013568:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231579.866997:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231580.891180:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231581.915143:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231582.939161:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231583.963109:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231584.539118:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231584.539133:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231584.539143:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231584.539155:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4456 00000100:00080000:1.0:1761231584.539160:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231584.539169:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231584.539217:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607013632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231584.539254:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231584.539259:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231584.539260:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231584.539265:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4456 00000100:00080000:1.0:1761231584.539268:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231584.539271:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231584.539280:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231584.539293:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607013696/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231584.539303:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231584.539307:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231584.539309:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231584.539312:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4456 00000100:00080000:1.0:1761231584.539314:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231584.539317:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231584.539321:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231584.539330:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607013760/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231584.539341:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231584.539345:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231584.539346:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231584.539351:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4456 00000100:00080000:1.0:1761231584.539353:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231584.539359:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231584.539363:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231584.539377:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607013824/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231584.539423:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607013632:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231584.539586:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231584/real 1761231584] req@00000000b72feabb x1846780607013632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231609 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231584.539610:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607013632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231609 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231584.539625:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231584.539630:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231584.539636:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607013632:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231584.539645:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013696:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231584.539674:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231584/real 1761231584] req@00000000659d78e3 x1846780607013696/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231609 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231584.539686:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607013696/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231609 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231584.539695:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231584.539698:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231584.539701:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013696:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231584.539708:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013760:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231584.539737:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231584/real 1761231584] req@00000000db10af3c x1846780607013760/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231609 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231584.539749:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607013760/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231609 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231584.539759:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231584.539761:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231584.539764:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013760:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231584.539770:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013824:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231584.539791:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231584/real 1761231584] req@00000000d07717c4 x1846780607013824/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231609 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231584.539802:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607013824/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231609 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231584.539812:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231584.539815:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231584.539818:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013824:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231584.987214:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231586.011132:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231587.035156:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231588.058979:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231589.083141:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231589.659091:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231589.659104:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231589.659109:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231589.659122:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4462 00000100:00080000:3.0:1761231589.659126:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231589.659135:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231589.659179:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607013888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231589.659217:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231589.659221:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231589.659223:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231589.659228:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4462 00000100:00080000:3.0:1761231589.659230:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231589.659234:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231589.659244:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231589.659256:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607013952/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231589.659271:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231589.659276:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231589.659278:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231589.659282:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4462 00000100:00080000:3.0:1761231589.659284:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231589.659288:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231589.659294:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231589.659308:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607013888:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231589.659313:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607014016/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231589.659331:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231589.659335:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231589.659337:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231589.659341:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4462 00000100:00080000:3.0:1761231589.659344:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231589.659347:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231589.659352:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231589.659366:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607014080/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231589.659438:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013952:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231589.659502:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014016:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231589.659542:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014080:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:3.0:1761231590.107124:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231591.131076:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231592.155127:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231592.731671:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231589/real 1761231592] req@000000002ce934d3 x1846780607014016/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231614 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231592.731717:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607014016/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231614 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231592.731732:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231592.731736:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231592.731741:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014016:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231592.731749:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231589/real 1761231592] req@000000006572b591 x1846780607014080/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231614 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231592.731762:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607014080/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231614 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231592.731770:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231592.731773:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231592.731776:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014080:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231592.731783:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231589/real 1761231592] req@00000000fc01c41c x1846780607013888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231614 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231592.731795:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607013888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231614 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231592.731803:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231592.731806:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231592.731810:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607013888:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231592.731817:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231589/real 1761231592] req@00000000408f088c x1846780607013952/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231614 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231592.731923:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607013952/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231614 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231592.731934:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231592.731943:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231592.731954:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607013952:192.168.201.104@tcp:38:kworker/u8:0.0 00010000:00010000:0.0:1761231593.179149:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231594.203157:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231594.779073:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231594.779090:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231594.779097:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231594.779113:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4467 00000100:00080000:2.0:1761231594.779120:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231594.779129:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231594.779181:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607014144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231594.779233:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231594.779240:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231594.779243:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231594.779250:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4467 00000100:00080000:2.0:1761231594.779254:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231594.779260:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231594.779275:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231594.779299:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607014208/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231594.779318:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231594.779326:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231594.779330:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231594.779338:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4467 00000100:00080000:2.0:1761231594.779342:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231594.779350:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231594.779360:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231594.779378:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607014144:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761231594.779388:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607014272/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231594.779433:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231594.779440:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231594.779443:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231594.779449:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4467 00000100:00080000:2.0:1761231594.779452:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231594.779459:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231594.779467:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231594.779490:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607014336/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231594.779569:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231594/real 1761231594] req@0000000017386078 x1846780607014144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231619 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231594.779604:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607014144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231619 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231594.779623:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231594.779629:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231594.779637:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607014144:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231594.779652:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014208:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231594.779707:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231594/real 1761231594] req@00000000f38f27df x1846780607014208/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231619 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231594.779732:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607014208/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231619 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231594.779747:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231594.779753:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231594.779759:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014208:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231594.779918:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014272:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231594.779993:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231594/real 1761231594] req@00000000c069b612 x1846780607014272/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231619 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231594.780018:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607014272/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231619 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231594.780032:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231594.780037:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231594.780043:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014272:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231594.780054:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014336:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231594.780105:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231594/real 1761231594] req@0000000094135f44 x1846780607014336/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231619 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231594.780155:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607014336/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231619 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231594.780171:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231594.780175:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231594.780181:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014336:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761231595.227092:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231596.251103:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231597.275152:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231598.299068:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231599.323014:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231599.898990:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231599.899002:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231599.899007:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231599.899018:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4472 00000100:00080000:2.0:1761231599.899022:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231599.899029:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231599.899069:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607014400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231599.899096:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231599.899099:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231599.899100:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231599.899103:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4472 00000100:00080000:2.0:1761231599.899105:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231599.899106:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231599.899115:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231599.899125:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607014464/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231599.899131:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231599.899133:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231599.899134:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231599.899137:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4472 00000100:00080000:2.0:1761231599.899138:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231599.899139:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231599.899142:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231599.899148:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607014528/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231599.899152:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231599.899155:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231599.899156:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231599.899159:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4472 00000100:00080000:2.0:1761231599.899160:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231599.899163:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231599.899165:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231599.899170:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607014592/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231599.899217:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607014400:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231599.899304:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231599/real 1761231599] req@0000000091106a97 x1846780607014400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231624 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231599.899317:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607014400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231624 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231599.899324:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231599.899327:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231599.899330:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607014400:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231599.899335:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014464:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231599.899349:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231599/real 1761231599] req@0000000064978143 x1846780607014464/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231624 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231599.899356:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607014464/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231624 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231599.899360:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231599.899361:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231599.899363:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014464:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231599.899366:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014528:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231599.899377:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231599/real 1761231599] req@0000000094135f44 x1846780607014528/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231624 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231599.899383:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607014528/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231624 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231599.899389:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231599.899390:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231599.899392:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014528:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231599.899395:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014592:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231599.899405:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231599/real 1761231599] req@00000000c069b612 x1846780607014592/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231624 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231599.899411:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607014592/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231624 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231599.899415:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231599.899416:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231599.899417:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014592:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761231600.346994:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231601.371096:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231602.395069:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231603.419137:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231604.443185:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231605.019090:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231605.019105:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231605.019110:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231605.019123:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4477 00000100:00080000:1.0:1761231605.019128:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231605.019136:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231605.019187:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607014656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231605.019224:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231605.019229:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231605.019231:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231605.019235:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4477 00000100:00080000:1.0:1761231605.019238:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231605.019241:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231605.019254:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231605.019267:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607014720/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231605.019277:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231605.019281:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231605.019282:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231605.019286:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4477 00000100:00080000:1.0:1761231605.019287:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231605.019291:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231605.019295:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231605.019304:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607014784/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231605.019315:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231605.019319:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231605.019321:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231605.019325:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4477 00000100:00080000:1.0:1761231605.019327:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231605.019330:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231605.019335:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231605.019345:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607014848/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231605.019353:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607014656:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231605.019468:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231605/real 1761231605] req@00000000d07717c4 x1846780607014656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231630 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231605.019490:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607014656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231630 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231605.019502:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231605.019506:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231605.019511:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607014656:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231605.019520:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014720:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231605.019546:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231605/real 1761231605] req@00000000db10af3c x1846780607014720/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231630 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231605.019559:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607014720/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231630 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231605.019568:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231605.019570:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231605.019573:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014720:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231605.019579:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014784:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231605.019601:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231605/real 1761231605] req@00000000659d78e3 x1846780607014784/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231630 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231605.019612:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607014784/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231630 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231605.019621:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231605.019624:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231605.019626:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014784:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231605.019701:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014848:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231605.019732:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231605/real 1761231605] req@00000000b72feabb x1846780607014848/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231630 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231605.019744:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607014848/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231630 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231605.019754:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231605.019756:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231605.019760:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014848:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231605.467026:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231606.491153:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231607.514983:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231608.539057:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231609.563028:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231610.138976:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231610.138990:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231610.138994:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231610.139009:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4482 00000100:00080000:3.0:1761231610.139013:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231610.139022:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231610.139072:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607014912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231610.139101:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231610.139105:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231610.139107:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231610.139111:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4482 00000100:00080000:3.0:1761231610.139113:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231610.139116:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231610.139126:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231610.139137:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607014976/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231610.139146:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231610.139149:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231610.139150:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231610.139154:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4482 00000100:00080000:3.0:1761231610.139157:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231610.139160:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231610.139165:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231610.139177:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607015040/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231610.139184:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231610.139188:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231610.139189:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231610.139193:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4482 00000100:00080000:3.0:1761231610.139196:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231610.139198:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231610.139202:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231610.139209:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607014912:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231610.139219:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607015104/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231610.139303:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231610/real 1761231610] req@00000000408f088c x1846780607014912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231635 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231610.139317:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607014912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231635 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231610.139325:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231610.139328:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231610.139330:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607014912:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231610.139335:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014976:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231610.139350:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231610/real 1761231610] req@00000000fc01c41c x1846780607014976/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231635 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231610.139357:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607014976/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231635 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231610.139361:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231610.139363:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231610.139366:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607014976:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231610.139369:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015040:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231610.139381:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231610/real 1761231610] req@000000006572b591 x1846780607015040/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231635 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231610.139386:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607015040/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231635 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231610.139391:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231610.139392:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231610.139394:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015040:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231610.139416:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015104:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231610.139428:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231610/real 1761231610] req@000000002ce934d3 x1846780607015104/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231635 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231610.139433:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607015104/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231635 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231610.139438:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231610.139439:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231610.139441:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015104:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231610.587031:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231611.611018:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231612.635002:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231613.659034:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231614.683003:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231615.258926:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231615.258936:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231615.258940:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231615.258949:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4487 00000100:00080000:3.0:1761231615.258953:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231615.258959:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231615.258993:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607015168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231615.259019:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231615.259023:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231615.259024:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231615.259028:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4487 00000100:00080000:3.0:1761231615.259030:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231615.259033:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231615.259041:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231615.259051:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607015232/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231615.259060:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231615.259063:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231615.259065:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231615.259068:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4487 00000100:00080000:3.0:1761231615.259070:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231615.259073:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231615.259078:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231615.259089:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607015296/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231615.259097:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607015168:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761231615.259100:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231615.259105:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231615.259106:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231615.259111:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4487 00000100:00080000:3.0:1761231615.259113:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231615.259116:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231615.259121:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231615.259130:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607015360/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231615.259176:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231615/real 1761231615] req@0000000090b40c66 x1846780607015168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231640 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231615.259188:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607015168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231640 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231615.259194:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231615.259196:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231615.259199:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607015168:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231615.259203:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015232:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231615.259218:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231615/real 1761231615] req@000000001f8e688e x1846780607015232/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231640 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231615.259225:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607015232/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231640 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231615.259229:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231615.259230:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231615.259231:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015232:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231615.259249:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015296:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231615.259262:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231615/real 1761231615] req@00000000a58ca11e x1846780607015296/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231640 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231615.259267:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607015296/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231640 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231615.259271:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231615.259273:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231615.259274:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015296:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231615.259277:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015360:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231615.259288:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231615/real 1761231615] req@0000000083f16a6b x1846780607015360/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231640 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231615.259293:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607015360/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231640 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231615.259297:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231615.259298:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231615.259300:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015360:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231615.707029:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231616.731018:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231617.755065:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231618.778978:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231619.803031:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231620.378905:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231620.378915:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231620.378917:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231620.378927:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4492 00000100:00080000:3.0:1761231620.378930:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231620.378936:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231620.378967:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607015424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231620.378989:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231620.378993:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231620.378994:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231620.378998:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4492 00000100:00080000:3.0:1761231620.379000:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231620.379004:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231620.379013:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231620.379024:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607015488/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231620.379029:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607015424:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761231620.379033:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231620.379036:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231620.379037:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231620.379040:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4492 00000100:00080000:3.0:1761231620.379042:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231620.379045:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231620.379048:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231620.379058:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607015552/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231620.379065:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231620.379068:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231620.379069:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231620.379072:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4492 00000100:00080000:3.0:1761231620.379074:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231620.379077:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231620.379080:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231620.379087:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607015616/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231620.379111:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231620/real 1761231620] req@0000000031e043f4 x1846780607015424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231645 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231620.379128:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607015424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231645 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231620.379139:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231620.379141:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231620.379144:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607015424:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231620.379162:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015488:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231620.379183:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231620/real 1761231620] req@000000007361665d x1846780607015488/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231645 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231620.379190:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607015488/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231645 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231620.379195:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231620.379197:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231620.379199:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015488:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231620.379202:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015552:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231620.379214:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231620/real 1761231620] req@00000000f87d0618 x1846780607015552/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231645 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231620.379220:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607015552/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231645 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231620.379225:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231620.379226:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231620.379228:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015552:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231620.379231:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015616:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231620.379242:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231620/real 1761231620] req@00000000c7e4e432 x1846780607015616/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231645 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231620.379248:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607015616/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231645 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231620.379253:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231620.379254:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231620.379256:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015616:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231620.827012:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231621.851007:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231622.875088:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231623.899012:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231624.923003:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231625.498894:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231625.498903:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231625.498906:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231625.498915:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4497 00000100:00080000:3.0:1761231625.498918:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231625.498924:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231625.498953:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607015680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231625.498971:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231625.498974:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231625.498975:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231625.498977:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4497 00000100:00080000:3.0:1761231625.498979:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231625.498981:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231625.498988:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231625.498996:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607015744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231625.499002:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231625.499006:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1761231625.499007:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607015680:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761231625.499008:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231625.499014:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4497 00000100:00080000:3.0:1761231625.499016:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231625.499018:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231625.499021:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231625.499030:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607015808/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231625.499035:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231625.499037:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231625.499038:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231625.499041:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4497 00000100:00080000:3.0:1761231625.499042:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231625.499044:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231625.499047:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231625.499053:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607015872/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231625.499080:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231625/real 1761231625] req@00000000bd684ccc x1846780607015680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231650 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231625.499093:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607015680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231650 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231625.499100:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231625.499102:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231625.499105:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607015680:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231625.499122:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015744:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231625.499138:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231625/real 1761231625] req@0000000018a8f5da x1846780607015744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231650 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231625.499145:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607015744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231650 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231625.499149:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231625.499150:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231625.499152:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015744:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231625.499155:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015808:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231625.499167:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231625/real 1761231625] req@000000005a9bdefd x1846780607015808/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231650 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231625.499173:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607015808/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231650 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231625.499178:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231625.499180:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231625.499181:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015808:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231625.499184:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015872:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231625.499206:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231625/real 1761231625] req@0000000086ccb527 x1846780607015872/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231650 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231625.499213:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607015872/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231650 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231625.499218:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231625.499220:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231625.499221:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607015872:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231625.946983:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231626.971161:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231627.995155:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231629.019041:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231630.043139:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231630.619002:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231630.619015:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231630.619019:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231630.619030:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4503 00000100:00080000:3.0:1761231630.619034:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231630.619042:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231630.619082:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607015936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231630.619140:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231630.619147:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231630.619150:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231630.619158:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4503 00000100:00080000:3.0:1761231630.619162:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231630.619184:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231630.619197:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231630.619218:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607016000/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231630.619236:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231630.619242:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231630.619245:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231630.619252:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4503 00000100:00080000:3.0:1761231630.619255:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231630.619288:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231630.619295:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231630.619328:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607016064/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231630.619339:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607015936:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761231630.619342:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231630.619353:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231630.619356:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231630.619360:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4503 00000100:00080000:3.0:1761231630.619363:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231630.619366:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231630.619371:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231630.619383:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607016128/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231630.619472:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231630/real 1761231630] req@0000000088473b3e x1846780607015936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231655 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231630.619512:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607015936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231655 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231630.619528:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231630.619533:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231630.619539:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607015936:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231630.619551:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016000:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231630.619599:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231630/real 1761231630] req@000000004e3f612a x1846780607016000/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231655 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231630.619619:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607016000/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231655 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231630.619632:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231630.619652:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231630.619657:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016000:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231630.619699:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016064:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231630.619749:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231630/real 1761231630] req@00000000daa39bb1 x1846780607016064/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231655 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231630.619768:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607016064/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231655 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231630.619783:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231630.619810:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231630.619815:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016064:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231630.619825:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016128:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231630.619866:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231630/real 1761231630] req@000000003ad149cc x1846780607016128/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231655 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231630.619934:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607016128/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231655 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231630.619948:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231630.619951:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231630.619957:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016128:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231631.067063:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231632.091112:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231633.115148:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231634.139152:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231635.163181:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231635.738922:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231635.738930:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231635.738934:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231635.738943:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4508 00000100:00080000:3.0:1761231635.738947:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231635.738955:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231635.738995:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607016192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231635.739027:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231635.739031:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231635.739033:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231635.739037:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4508 00000100:00080000:3.0:1761231635.739039:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231635.739043:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231635.739052:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231635.739065:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607016256/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231635.739075:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231635.739078:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231635.739079:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231635.739083:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4508 00000100:00080000:3.0:1761231635.739085:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231635.739088:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231635.739092:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231635.739100:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607016320/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231635.739107:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761231635.739109:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607016192:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761231635.739110:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231635.739112:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231635.739119:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4508 00000100:00080000:3.0:1761231635.739121:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231635.739123:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231635.739125:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231635.739148:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607016384/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231635.739226:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231635/real 1761231635] req@000000003ad149cc x1846780607016192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231660 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231635.739239:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607016192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231660 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231635.739246:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231635.739248:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231635.739251:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607016192:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231635.739256:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016256:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231635.739272:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231635/real 1761231635] req@00000000daa39bb1 x1846780607016256/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231660 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231635.739280:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607016256/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231660 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231635.739284:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231635.739285:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231635.739286:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016256:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231635.739305:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016320:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231635.739318:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231635/real 1761231635] req@000000004e3f612a x1846780607016320/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231660 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231635.739324:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607016320/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231660 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231635.739329:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231635.739330:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231635.739331:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016320:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231635.739335:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016384:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231635.739345:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231635/real 1761231635] req@0000000088473b3e x1846780607016384/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231660 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231635.739351:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607016384/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231660 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231635.739356:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231635.739357:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231635.739359:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016384:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231636.187167:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231637.211023:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231638.235026:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231639.259024:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231640.283147:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231640.858939:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231640.858948:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231640.858951:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231640.858961:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4513 00000100:00080000:3.0:1761231640.858964:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231640.858970:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231640.859003:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607016448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231640.859024:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231640.859028:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231640.859029:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231640.859032:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4513 00000100:00080000:3.0:1761231640.859033:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231640.859035:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231640.859042:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231640.859049:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607016512/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231640.859055:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231640.859057:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231640.859058:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231640.859061:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4513 00000100:00080000:3.0:1761231640.859063:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231640.859065:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231640.859068:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231640.859078:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607016448:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231640.859079:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607016576/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231640.859091:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231640.859092:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231640.859093:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231640.859095:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4513 00000100:00080000:3.0:1761231640.859097:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231640.859099:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231640.859101:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231640.859132:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607016640/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231640.859185:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231640/real 1761231640] req@0000000086ccb527 x1846780607016448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231665 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231640.859199:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607016448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231665 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231640.859206:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231640.859208:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231640.859211:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607016448:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231640.859216:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016512:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231640.859232:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231640/real 1761231640] req@000000005a9bdefd x1846780607016512/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231665 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231640.859240:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607016512/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231665 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231640.859244:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231640.859245:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231640.859247:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016512:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231640.859266:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016576:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231640.859282:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231640/real 1761231640] req@0000000018a8f5da x1846780607016576/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231665 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231640.859289:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607016576/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231665 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231640.859294:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231640.859295:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231640.859297:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016576:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231640.859300:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016640:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231640.859311:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231640/real 1761231640] req@00000000bd684ccc x1846780607016640/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231665 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231640.859317:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607016640/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231665 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231640.859322:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231640.859323:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231640.859325:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016640:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231641.307003:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231642.331180:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231643.355047:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231644.379186:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231645.403223:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231645.979065:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231645.979078:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231645.979084:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231645.979096:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4518 00000100:00080000:2.0:1761231645.979101:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231645.979109:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231645.979158:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607016704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231645.979197:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231645.979201:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231645.979203:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231645.979208:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4518 00000100:00080000:2.0:1761231645.979211:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231645.979214:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231645.979224:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231645.979236:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607016768/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231645.979248:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231645.979252:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231645.979255:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231645.979260:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4518 00000100:00080000:2.0:1761231645.979262:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231645.979268:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231645.979274:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231645.979286:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607016832/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231645.979293:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607016704:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761231645.979306:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231645.979312:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231645.979314:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231645.979319:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4518 00000100:00080000:2.0:1761231645.979321:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231645.979325:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231645.979331:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231645.979344:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607016896/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231645.979416:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231645/real 1761231645] req@00000000f38f27df x1846780607016704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231670 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231645.979446:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607016704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231670 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231645.979458:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231645.979461:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231645.979466:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607016704:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231645.979475:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016768:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231645.979503:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231645/real 1761231645] req@0000000017386078 x1846780607016768/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231670 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231645.979515:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607016768/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231670 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231645.979523:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231645.979525:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231645.979528:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016768:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231645.979557:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016832:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231645.979581:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231645/real 1761231645] req@00000000c069b612 x1846780607016832/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231670 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231645.979593:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607016832/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231670 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231645.979602:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231645.979604:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231645.979607:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016832:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231645.979614:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016896:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231645.979634:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231645/real 1761231645] req@0000000094135f44 x1846780607016896/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231670 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231645.979646:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607016896/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231670 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231645.979655:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231645.979657:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231645.979660:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607016896:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231646.427173:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231647.451105:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231648.475147:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231649.499094:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231650.523145:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231651.098994:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231651.099003:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231651.099007:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231651.099016:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4523 00000100:00080000:3.0:1761231651.099019:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231651.099025:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231651.099057:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607016960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231651.099080:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231651.099083:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231651.099084:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231651.099087:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4523 00000100:00080000:3.0:1761231651.099088:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231651.099090:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231651.099096:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231651.099103:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607017024/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231651.099108:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231651.099110:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231651.099111:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231651.099113:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4523 00000100:00080000:3.0:1761231651.099114:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231651.099116:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231651.099120:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231651.099126:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607017088/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231651.099131:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231651.099133:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231651.099134:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231651.099136:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4523 00000100:00080000:3.0:1761231651.099137:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231651.099139:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231651.099141:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231651.099147:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607017152/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231651.099166:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607016960:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231651.099244:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231651/real 1761231651] req@00000000c7e4e432 x1846780607016960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231676 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231651.099256:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607016960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231676 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231651.099263:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231651.099265:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231651.099268:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607016960:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231651.099272:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017024:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231651.099286:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231651/real 1761231651] req@00000000f87d0618 x1846780607017024/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231676 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231651.099293:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607017024/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231676 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231651.099297:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231651.099299:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231651.099300:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017024:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231651.099303:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017088:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231651.099314:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231651/real 1761231651] req@000000007361665d x1846780607017088/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231676 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231651.099320:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607017088/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231676 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231651.099325:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231651.099326:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231651.099327:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017088:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231651.099330:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017152:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231651.099340:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231651/real 1761231651] req@0000000031e043f4 x1846780607017152/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231676 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231651.099346:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607017152/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231676 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231651.099350:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231651.099351:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231651.099354:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017152:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231651.547016:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231652.571052:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231653.595139:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231654.619216:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231655.643157:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231656.219017:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231656.219026:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231656.219030:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231656.219039:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4528 00000100:00080000:3.0:1761231656.219042:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231656.219048:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231656.219082:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607017216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231656.219108:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231656.219110:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231656.219112:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231656.219115:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4528 00000100:00080000:3.0:1761231656.219117:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231656.219120:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231656.219132:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231656.219140:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607017280/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231656.219146:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231656.219149:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231656.219150:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231656.219153:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4528 00000100:00080000:3.0:1761231656.219155:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231656.219157:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231656.219160:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231656.219169:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607017344/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231656.219175:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231656.219178:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231656.219179:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231656.219184:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4528 00000100:00100000:0.0:1761231656.219185:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607017216:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761231656.219185:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231656.219189:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231656.219193:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231656.219201:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607017408/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231656.219278:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017280:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231656.219304:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017344:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231656.219318:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017408:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231656.667141:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231657.691096:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231658.715036:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231659.291193:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231656/real 1761231659] req@000000001f8e688e x1846780607017344/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231681 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231659.291222:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607017344/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231681 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231659.291235:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231659.291239:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231659.291244:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017344:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231659.291250:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231656/real 1761231659] req@0000000090b40c66 x1846780607017408/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231681 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231659.291261:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607017408/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231681 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231659.291270:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231659.291272:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231659.291275:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017408:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231659.291281:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231656/real 1761231659] req@0000000083f16a6b x1846780607017216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231681 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231659.291292:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607017216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231681 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231659.291299:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231659.291301:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231659.291304:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607017216:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231659.291309:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231656/real 1761231659] req@00000000a58ca11e x1846780607017280/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231681 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231659.291318:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607017280/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231681 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231659.291325:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231659.291327:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231659.291330:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017280:192.168.201.104@tcp:38:kworker/u8:2.0 00010000:00010000:0.0:1761231659.739132:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231660.763041:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231661.338993:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231661.339011:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231661.339017:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231661.339035:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4533 00000100:00080000:3.0:1761231661.339042:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231661.339053:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231661.339104:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607017472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231661.339154:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231661.339161:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231661.339164:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231661.339172:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4533 00000100:00080000:3.0:1761231661.339177:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231661.339185:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231661.339200:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231661.339215:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607017472:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231661.339229:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607017536/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231661.339253:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231661.339259:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231661.339262:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231661.339269:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4533 00000100:00080000:3.0:1761231661.339273:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231661.339281:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231661.339290:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231661.339315:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607017600/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231661.339335:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231661.339342:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231661.339345:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1761231661.339349:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231661/real 1761231661] req@000000002ce934d3 x1846780607017472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231686 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231661.339359:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4533 00000100:00080000:3.0:1761231661.339368:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231661.339380:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231661.339393:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231661.339424:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607017664/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231661.339426:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607017472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231686 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231661.339477:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231661.339486:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231661.339495:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607017472:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231661.339551:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017536:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231661.339603:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231661/real 1761231661] req@000000006572b591 x1846780607017536/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231686 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231661.339622:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607017536/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231686 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231661.339635:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231661.339640:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231661.339645:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017536:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231661.339654:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017600:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231661.339697:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231661/real 1761231661] req@00000000fc01c41c x1846780607017600/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231686 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231661.339717:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607017600/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231686 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231661.339730:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231661.339734:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231661.339739:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017600:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231661.339749:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017664:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231661.339788:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231661/real 1761231661] req@00000000408f088c x1846780607017664/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231686 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231661.339894:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607017664/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231686 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231661.339908:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231661.339911:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231661.339916:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017664:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761231661.788005:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231662.811151:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231663.835048:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231664.859066:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231665.883123:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231666.458938:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231666.458948:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231666.458951:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231666.458961:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4538 00000100:00080000:3.0:1761231666.458964:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231666.458970:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231666.459004:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607017728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231666.459026:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231666.459029:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231666.459030:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231666.459033:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4538 00000100:00080000:3.0:1761231666.459034:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231666.459036:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231666.459044:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231666.459052:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607017792/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231666.459058:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231666.459060:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231666.459061:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231666.459063:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4538 00000100:00080000:3.0:1761231666.459065:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231666.459066:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231666.459069:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231666.459075:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607017856/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231666.459080:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231666.459081:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231666.459082:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231666.459084:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4538 00000100:00080000:3.0:1761231666.459086:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231666.459088:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231666.459090:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231666.459097:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607017920/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231666.459117:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607017728:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231666.459193:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231666/real 1761231666] req@00000000408f088c x1846780607017728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231691 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231666.459206:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607017728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231691 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231666.459214:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231666.459216:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231666.459220:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607017728:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231666.459225:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017792:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231666.459240:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231666/real 1761231666] req@00000000fc01c41c x1846780607017792/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231691 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231666.459252:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607017792/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231691 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231666.459257:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231666.459258:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231666.459260:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017792:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231666.459263:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017856:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231666.459276:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231666/real 1761231666] req@000000006572b591 x1846780607017856/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231691 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231666.459282:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607017856/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231691 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231666.459288:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231666.459289:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231666.459291:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017856:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231666.459294:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017920:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231666.459306:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231666/real 1761231666] req@000000002ce934d3 x1846780607017920/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231691 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231666.459312:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607017920/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231691 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231666.459318:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231666.459319:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231666.459321:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607017920:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:3.0:1761231666.907033:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231667.931117:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231668.955180:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231669.979211:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231671.003233:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231671.578957:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231671.578972:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231671.578978:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231671.578990:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4543 00000100:00080000:3.0:1761231671.578994:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231671.579004:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231671.579051:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607017984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231671.579086:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231671.579091:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231671.579093:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231671.579097:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4543 00000100:00080000:3.0:1761231671.579100:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231671.579103:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231671.579113:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231671.579125:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607018048/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231671.579135:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231671.579138:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231671.579140:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231671.579145:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4543 00000100:00080000:3.0:1761231671.579147:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231671.579151:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231671.579156:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231671.579170:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607018112/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231671.579182:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231671.579187:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231671.579189:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231671.579197:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4543 00000100:00100000:0.0:1761231671.579201:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607017984:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761231671.579202:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231671.579210:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231671.579220:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231671.579235:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607018176/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231671.579349:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231671/real 1761231671] req@00000000a58ca11e x1846780607017984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231696 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231671.579381:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607017984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231696 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231671.579399:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231671.579404:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231671.579410:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607017984:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231671.579423:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018048:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231671.579477:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231671/real 1761231671] req@0000000083f16a6b x1846780607018048/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231696 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231671.579500:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607018048/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231696 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231671.579513:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231671.579518:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231671.579524:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018048:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231671.579566:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018112:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231671.579618:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231671/real 1761231671] req@0000000090b40c66 x1846780607018112/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231696 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231671.579640:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607018112/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231696 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231671.579687:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231671.579692:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231671.579698:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018112:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231671.579710:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018176:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231671.579763:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231671/real 1761231671] req@000000001f8e688e x1846780607018176/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231696 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231671.579784:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607018176/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231696 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231671.579798:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231671.579803:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231671.579808:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018176:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231672.027178:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231673.051096:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231674.075216:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231675.099214:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231676.123211:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231676.699073:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231676.699087:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231676.699093:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231676.699105:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4549 00000100:00080000:2.0:1761231676.699110:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231676.699185:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231676.699232:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607018240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231676.699273:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231676.699278:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231676.699280:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231676.699284:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4549 00000100:00080000:2.0:1761231676.699287:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231676.699291:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231676.699301:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231676.699313:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607018304/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231676.699324:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231676.699327:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231676.699329:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231676.699333:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4549 00000100:00080000:2.0:1761231676.699335:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231676.699338:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231676.699342:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231676.699353:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607018368/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231676.699362:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231676.699366:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231676.699370:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231676.699374:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4549 00000100:00080000:2.0:1761231676.699377:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231676.699384:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231676.699390:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231676.699402:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607018432/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231676.699451:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607018240:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231676.699645:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231676/real 1761231676] req@0000000064978143 x1846780607018240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231701 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231676.699671:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607018240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231701 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231676.699687:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231676.699692:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231676.699697:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607018240:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231676.699707:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018304:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231676.699739:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231676/real 1761231676] req@0000000091106a97 x1846780607018304/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231701 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231676.699752:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607018304/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231701 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231676.699760:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231676.699764:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231676.699767:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018304:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231676.699773:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018368:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231676.699797:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231676/real 1761231676] req@0000000094135f44 x1846780607018368/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231701 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231676.699809:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607018368/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231701 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231676.699818:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231676.699821:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231676.699824:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018368:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231676.699884:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018432:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231676.699919:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231676/real 1761231676] req@00000000c069b612 x1846780607018432/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231701 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231676.699932:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607018432/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231701 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231676.699942:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231676.699945:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231676.699948:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018432:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231677.147125:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231678.171208:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231679.195185:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231680.219273:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231681.243133:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231681.818973:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231681.818992:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231681.818998:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231681.819016:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4554 00000100:00080000:2.0:1761231681.819022:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231681.819033:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231681.821018:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607018496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231681.821069:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231681.821076:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231681.821080:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231681.821086:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4554 00000100:00080000:2.0:1761231681.821090:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231681.821099:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231681.821127:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231681.821163:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607018560/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231681.821182:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231681.821189:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231681.821193:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231681.821203:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4554 00000100:00080000:2.0:1761231681.821207:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231681.821215:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231681.821226:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231681.821229:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607018496:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761231681.821254:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607018624/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231681.821277:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231681.821285:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231681.821288:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231681.821297:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4554 00000100:00080000:2.0:1761231681.821301:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231681.821309:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231681.821316:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231681.821336:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607018688/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231681.821355:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231681/real 1761231681] req@0000000017386078 x1846780607018496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231706 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231681.821380:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607018496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231706 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231681.821395:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231681.821399:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231681.821404:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607018496:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231681.821413:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018560:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231681.821441:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231681/real 1761231681] req@00000000f38f27df x1846780607018560/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231706 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231681.821463:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607018560/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231706 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231681.821471:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231681.821474:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231681.821477:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018560:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231681.821533:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018624:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231681.821566:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231681/real 1761231681] req@00000000c069b612 x1846780607018624/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231706 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231681.821579:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607018624/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231706 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231681.821589:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231681.821592:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231681.821595:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018624:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231681.821600:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018688:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231681.821621:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231681/real 1761231681] req@0000000094135f44 x1846780607018688/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231706 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231681.821632:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607018688/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231706 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231681.821641:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231681.821643:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231681.821647:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018688:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231682.267133:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231683.291286:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231684.315053:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231685.339194:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231686.363089:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231686.938982:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231686.938991:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231686.938994:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231686.939003:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4559 00000100:00080000:0.0:1761231686.939006:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231686.939012:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231686.939043:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607018752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231686.939067:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231686.939069:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231686.939070:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231686.939073:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4559 00000100:00080000:0.0:1761231686.939075:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231686.939077:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231686.939083:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231686.939089:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607018816/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231686.939095:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231686.939114:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231686.939116:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231686.939120:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4559 00000100:00080000:0.0:1761231686.939122:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231686.939124:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231686.939128:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231686.939137:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607018880/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231686.939145:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231686.939148:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231686.939149:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231686.939152:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4559 00000100:00080000:0.0:1761231686.939155:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231686.939157:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231686.939161:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231686.939168:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607018944/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231686.939176:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607018752:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231686.939270:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231686/real 1761231686] req@0000000099514377 x1846780607018752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231711 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231686.939287:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607018752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231711 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231686.939298:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231686.939300:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231686.939304:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607018752:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231686.939311:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018816:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231686.939331:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231686/real 1761231686] req@000000000243251f x1846780607018816/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231711 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231686.939340:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607018816/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231711 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231686.939347:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231686.939349:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231686.939352:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018816:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231686.939357:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018880:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231686.939375:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231686/real 1761231686] req@00000000271a9d8e x1846780607018880/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231711 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231686.939383:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607018880/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231711 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231686.939390:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231686.939392:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231686.939394:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018880:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231686.939443:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018944:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231686.939466:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231686/real 1761231686] req@0000000023ab3a3f x1846780607018944/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231711 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231686.939475:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607018944/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231711 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231686.939483:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231686.939484:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231686.939487:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607018944:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231687.387146:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231688.411205:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231689.435153:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231690.459148:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231691.483031:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231692.058973:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231692.059000:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231692.059006:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231692.059018:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4564 00000100:00080000:0.0:1761231692.059022:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231692.059031:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231692.059084:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607019008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231692.059117:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231692.059122:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231692.059124:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231692.059129:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4564 00000100:00080000:0.0:1761231692.059133:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231692.059141:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231692.059155:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231692.059180:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607019008:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231692.059184:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607019072/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231692.059211:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231692.059218:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231692.059221:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231692.059229:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4564 00000100:00080000:0.0:1761231692.059234:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231692.059240:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231692.059249:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231692.059280:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607019136/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231692.059293:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231692.059298:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231692.059301:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761231692.059310:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231692/real 1761231692] req@000000002ef1f54f x1846780607019008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231717 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231692.059311:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4564 00000100:00080000:0.0:1761231692.059318:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231692.059328:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231692.059365:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231692.059389:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607019200/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231692.059399:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607019008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231717 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231692.059420:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231692.059424:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231692.059430:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607019008:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231692.059462:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019072:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231692.059496:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231692/real 1761231692] req@00000000275aeee5 x1846780607019072/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231717 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231692.059509:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607019072/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231717 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231692.059527:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231692.059530:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231692.059533:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019072:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231692.059539:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019136:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231692.059561:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231692/real 1761231692] req@000000005a096598 x1846780607019136/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231717 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231692.059573:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607019136/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231717 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231692.059582:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231692.059585:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231692.059587:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019136:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231692.059594:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019200:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231692.059613:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231692/real 1761231692] req@0000000023ab3a3f x1846780607019200/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231717 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231692.059624:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607019200/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231717 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231692.059633:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231692.059635:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231692.059638:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019200:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231692.507110:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231693.531174:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231694.555037:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231695.579136:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231696.603141:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231697.178974:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231697.178986:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231697.178989:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231697.179000:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4569 00000100:00080000:2.0:1761231697.179003:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231697.179010:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231697.179046:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607019264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231697.179075:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231697.179080:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231697.179082:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231697.179086:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4569 00000100:00080000:2.0:1761231697.179088:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231697.179091:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231697.179099:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231697.179111:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607019328/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231697.179120:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231697.179122:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231697.179124:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231697.179127:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4569 00000100:00080000:2.0:1761231697.179129:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231697.179132:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231697.179138:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231697.179149:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607019392/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231697.179158:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231697.179161:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231697.179162:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231697.179167:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4569 00000100:00080000:2.0:1761231697.179169:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231697.179172:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231697.179177:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231697.179186:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607019264:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761231697.179192:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607019456/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231697.179329:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231697/real 1761231697] req@0000000091106a97 x1846780607019264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231722 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231697.179347:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607019264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231722 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231697.179358:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231697.179362:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231697.179366:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607019264:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231697.179373:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019328:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231697.179395:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231697/real 1761231697] req@0000000064978143 x1846780607019328/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231722 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231697.179404:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607019328/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231722 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231697.179416:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231697.179419:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231697.179421:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019328:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231697.179425:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019392:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231697.179441:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231697/real 1761231697] req@0000000094135f44 x1846780607019392/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231722 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231697.179449:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607019392/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231722 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231697.179456:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231697.179458:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231697.179460:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019392:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231697.179489:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019456:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231697.179505:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231697/real 1761231697] req@00000000c069b612 x1846780607019456/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231722 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231697.179512:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607019456/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231722 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231697.179519:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231697.179521:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231697.179524:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019456:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231697.627057:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231698.650970:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231699.675186:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231700.699200:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231701.723055:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231702.299087:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231702.299101:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231702.299106:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231702.299120:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4574 00000100:00080000:3.0:1761231702.299124:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231702.299133:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231702.299181:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607019520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231702.299246:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231702.299251:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231702.299253:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231702.299259:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4574 00000100:00080000:3.0:1761231702.299261:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231702.299265:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231702.299276:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231702.299290:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607019584/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231702.299302:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231702.299306:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231702.299308:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231702.299313:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4574 00000100:00080000:3.0:1761231702.299315:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231702.299322:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231702.299329:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231702.299339:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607019648/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231702.299350:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231702.299356:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231702.299358:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231702.299364:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4574 00000100:00080000:3.0:1761231702.299366:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231702.299370:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231702.299375:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231702.299388:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607019712/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231702.299391:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607019520:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231702.299581:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231702/real 1761231702] req@0000000031e043f4 x1846780607019520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231727 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231702.299608:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607019520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231727 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231702.299624:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231702.299629:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231702.299634:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607019520:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231702.299643:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019584:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231702.299679:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231702/real 1761231702] req@000000007361665d x1846780607019584/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231727 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231702.299692:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607019584/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231727 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231702.299701:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231702.299704:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231702.299707:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019584:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231702.299713:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019648:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231702.299742:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231702/real 1761231702] req@00000000f87d0618 x1846780607019648/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231727 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231702.299754:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607019648/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231727 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231702.299764:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231702.299766:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231702.299769:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019648:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231702.299812:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019712:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231702.299907:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231702/real 1761231702] req@00000000c7e4e432 x1846780607019712/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231727 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231702.299922:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607019712/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231727 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231702.299932:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231702.299937:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231702.299939:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019712:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231702.747112:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231703.771024:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231704.795099:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231705.819082:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231706.843156:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231707.419041:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231707.419064:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231707.419070:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231707.419087:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4579 00000100:00080000:0.0:1761231707.419093:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231707.419104:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231707.419207:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607019776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231707.419252:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231707.419259:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231707.419262:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231707.419270:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4579 00000100:00080000:0.0:1761231707.419273:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231707.419281:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231707.419297:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231707.419320:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607019840/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231707.419333:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607019776:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:0.0:1761231707.419336:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231707.419343:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231707.419347:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231707.419353:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4579 00000100:00080000:0.0:1761231707.419357:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231707.419362:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231707.419369:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231707.419385:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607019904/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231707.419399:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231707.419404:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231707.419406:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231707.419411:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4579 00000100:00080000:0.0:1761231707.419414:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231707.419419:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231707.419425:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231707.419438:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607019968/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231707.419498:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231707/real 1761231707] req@00000000271a9d8e x1846780607019776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231732 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231707.419531:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607019776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231732 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231707.419547:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231707.419551:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231707.419556:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607019776:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231707.419595:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019840:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231707.419635:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231707/real 1761231707] req@000000000243251f x1846780607019840/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231732 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231707.419648:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607019840/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231732 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231707.419659:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231707.419662:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231707.419665:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019840:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231707.419671:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019904:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231707.419694:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231707/real 1761231707] req@0000000099514377 x1846780607019904/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231732 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231707.419705:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607019904/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231732 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231707.419715:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231707.419718:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231707.419721:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019904:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231707.419726:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019968:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231707.419749:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231707/real 1761231707] req@00000000b8f2be78 x1846780607019968/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231732 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231707.419760:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607019968/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231732 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231707.419769:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231707.419772:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231707.419776:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607019968:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231707.867193:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231708.891130:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231709.915187:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231710.939037:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231711.963119:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231712.539006:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231712.539082:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231712.539086:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231712.539097:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4584 00000100:00080000:3.0:1761231712.539101:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231712.539108:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231712.539142:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607020032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231712.539171:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231712.539173:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231712.539175:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231712.539177:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4584 00000100:00080000:3.0:1761231712.539179:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231712.539182:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231712.539190:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231712.539199:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607020096/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231712.539206:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231712.539208:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231712.539209:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231712.539212:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4584 00000100:00080000:3.0:1761231712.539214:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231712.539221:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1761231712.539223:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607020032:192.168.201.104@tcp:250:kworker/u8:0.0 00000008:00100000:3.0:1761231712.539226:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231712.539238:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607020160/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231712.539245:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231712.539248:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231712.539249:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231712.539252:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4584 00000100:00080000:3.0:1761231712.539253:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231712.539255:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231712.539258:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231712.539265:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607020224/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231712.539344:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231712/real 1761231712] req@00000000bd684ccc x1846780607020032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231737 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231712.539372:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607020032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231737 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231712.539386:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231712.539389:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231712.539393:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607020032:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231712.539421:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020096:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231712.539438:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231712/real 1761231712] req@0000000018a8f5da x1846780607020096/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231737 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231712.539445:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607020096/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231737 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231712.539450:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231712.539451:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231712.539453:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020096:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231712.539479:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020160:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231712.539498:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231712/real 1761231712] req@000000005a9bdefd x1846780607020160/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231737 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231712.539506:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607020160/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231737 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231712.539511:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231712.539513:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231712.539515:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020160:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231712.539519:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020224:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231712.539532:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231712/real 1761231712] req@0000000086ccb527 x1846780607020224/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231737 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231712.539539:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607020224/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231737 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231712.539544:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231712.539545:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231712.539546:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020224:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231712.987146:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231714.011162:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231715.035246:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231716.059149:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231717.083159:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231717.658919:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231717.658928:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231717.658931:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231717.658940:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4590 00000100:00080000:3.0:1761231717.658943:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231717.658948:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231717.659003:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607020288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231717.659023:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231717.659026:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231717.659027:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231717.659030:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4590 00000100:00080000:3.0:1761231717.659031:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231717.659033:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231717.659042:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231717.659049:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607020352/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231717.659054:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231717.659056:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231717.659057:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231717.659059:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4590 00000100:00080000:3.0:1761231717.659060:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231717.659062:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231717.659064:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231717.659071:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607020416/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231717.659075:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231717.659077:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231717.659077:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231717.659079:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4590 00000100:00080000:3.0:1761231717.659080:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231717.659082:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231717.659085:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231717.659089:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607020480/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231717.659173:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607020288:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231717.659322:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231717/real 1761231717] req@0000000088473b3e x1846780607020288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231742 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231717.659346:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607020288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231742 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231717.659360:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231717.659365:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231717.659371:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607020288:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231717.659379:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020352:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231717.659406:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231717/real 1761231717] req@000000004e3f612a x1846780607020352/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231742 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231717.659429:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607020352/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231742 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231717.659437:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231717.659440:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231717.659443:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020352:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231717.659449:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020416:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231717.659486:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231717/real 1761231717] req@00000000daa39bb1 x1846780607020416/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231742 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231717.659499:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607020416/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231742 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231717.659508:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231717.659511:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231717.659514:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020416:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231717.659522:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020480:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231717.659545:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231717/real 1761231717] req@000000003ad149cc x1846780607020480/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231742 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231717.659557:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607020480/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231742 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231717.659566:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231717.659568:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231717.659572:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020480:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231718.107064:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231719.131232:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231720.155079:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231721.179211:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231722.203183:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231722.779015:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231722.779028:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231722.779033:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231722.779047:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4595 00000100:00080000:3.0:1761231722.779052:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231722.779060:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231722.779109:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607020544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231722.779142:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231722.779147:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231722.779149:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231722.779154:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4595 00000100:00080000:3.0:1761231722.779156:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231722.779159:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231722.779169:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231722.779182:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607020608/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231722.779192:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231722.779195:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231722.779196:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231722.779200:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4595 00000100:00080000:3.0:1761231722.779203:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231722.779206:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231722.779211:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231722.779223:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607020672/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231722.779233:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231722.779238:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231722.779240:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231722.779244:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4595 00000100:00080000:3.0:1761231722.779247:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231722.779250:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231722.779255:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231722.779266:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607020544:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231722.779269:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607020736/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231722.779391:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020608:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231722.779417:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020672:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231722.779450:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020736:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231723.227175:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231724.251155:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231725.275108:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231725.851457:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231722/real 1761231725] req@0000000088473b3e x1846780607020736/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231747 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231725.851486:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607020736/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231747 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231725.851497:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231725.851501:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231725.851505:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020736:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231725.851510:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231722/real 1761231725] req@000000003ad149cc x1846780607020544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231747 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231725.851524:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607020544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231747 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231725.851541:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231725.851545:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231725.851548:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607020544:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231725.851555:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231722/real 1761231725] req@00000000daa39bb1 x1846780607020608/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231747 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231725.851568:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607020608/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231747 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231725.851577:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231725.851580:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231725.851583:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020608:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231725.851589:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231722/real 1761231725] req@000000004e3f612a x1846780607020672/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231747 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231725.851601:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607020672/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231747 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231725.851611:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231725.851613:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231725.851617:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020672:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:2.0:1761231726.299149:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231727.323124:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231727.899165:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231727.899183:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231727.899189:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231727.899206:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4600 00000100:00080000:0.0:1761231727.899447:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231727.899461:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231727.899507:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607020800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231727.899567:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231727.899575:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231727.899580:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231727.899588:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4600 00000100:00080000:0.0:1761231727.899591:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231727.899596:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231727.899609:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231727.899627:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607020864/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231727.899642:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231727.899646:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231727.899649:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231727.899654:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4600 00000100:00080000:0.0:1761231727.899657:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231727.899662:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231727.899671:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231727.899691:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607020928/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231727.899706:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231727.899712:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231727.899716:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231727.899724:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4600 00000100:00080000:0.0:1761231727.899727:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231727.899737:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1761231727.899741:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607020800:192.168.201.104@tcp:250:kworker/u8:2.0 00000008:00100000:0.0:1761231727.899750:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231727.899941:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607020992/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231727.900278:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231727/real 1761231727] req@0000000001b2f813 x1846780607020800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231752 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231727.900313:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607020800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231752 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231727.900330:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231727.900336:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231727.900343:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607020800:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231727.900497:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020864:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231727.900604:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231727/real 1761231727] req@000000007f60e210 x1846780607020864/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231752 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231727.900627:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607020864/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231752 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231727.900639:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231727.900645:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231727.900650:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020864:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231727.900660:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020928:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231727.900707:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231727/real 1761231727] req@00000000fcce404c x1846780607020928/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231752 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231727.900726:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607020928/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231752 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231727.900739:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231727.900743:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231727.900747:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020928:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231727.900789:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020992:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231727.900904:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231727/real 1761231727] req@00000000b8f2be78 x1846780607020992/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231752 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231727.900922:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607020992/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231752 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231727.900937:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231727.900940:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231727.900944:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607020992:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231728.347071:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231729.371136:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231730.395124:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231731.419085:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231732.443134:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231733.019054:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231733.019069:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231733.019073:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231733.019087:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4605 00000100:00080000:3.0:1761231733.019091:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231733.019100:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231733.019147:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607021056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231733.019192:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231733.019197:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231733.019199:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231733.019204:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4605 00000100:00080000:3.0:1761231733.019206:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231733.019210:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231733.019221:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231733.019236:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607021120/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231733.019248:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231733.019252:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231733.019253:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231733.019257:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4605 00000100:00080000:3.0:1761231733.019259:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231733.019263:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231733.019271:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231733.019284:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607021056:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231733.019286:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607021184/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231733.019318:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231733.019324:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231733.019325:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231733.019330:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4605 00000100:00080000:3.0:1761231733.019334:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231733.019339:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231733.019343:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231733.019367:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607021248/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231733.019478:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231733/real 1761231733] req@0000000086ccb527 x1846780607021056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231758 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231733.019505:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607021056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231758 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231733.019520:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231733.019525:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231733.019531:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607021056:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231733.019541:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021120:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231733.019570:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231733/real 1761231733] req@000000005a9bdefd x1846780607021120/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231758 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231733.019583:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607021120/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231758 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231733.019600:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231733.019603:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231733.019606:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021120:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231733.019645:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021184:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231733.019671:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231733/real 1761231733] req@0000000018a8f5da x1846780607021184/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231758 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231733.019682:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607021184/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231758 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231733.019692:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231733.019694:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231733.019697:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021184:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231733.019703:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021248:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231733.019723:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231733/real 1761231733] req@00000000bd684ccc x1846780607021248/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231758 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231733.019767:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607021248/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231758 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231733.019777:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231733.019780:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231733.019783:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021248:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231733.467146:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231734.491077:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231735.515156:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231736.539133:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231737.563162:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231738.139081:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231738.139099:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231738.139105:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231738.139122:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4610 00000100:00080000:2.0:1761231738.139128:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231738.139139:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231738.139201:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607021312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231738.139246:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231738.139252:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231738.139254:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231738.139260:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4610 00000100:00080000:2.0:1761231738.139263:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231738.139266:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231738.139279:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231738.139292:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607021376/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231738.139312:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231738.139316:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231738.139318:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231738.139322:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4610 00000100:00080000:2.0:1761231738.139324:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231738.139327:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231738.139332:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231738.139347:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607021440/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231738.139356:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231738.139360:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231738.139361:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231738.139365:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4610 00000100:00080000:2.0:1761231738.139368:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231738.139373:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231738.139379:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231738.139394:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607021504/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231738.139403:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607021312:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231738.139540:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231738/real 1761231738] req@00000000f38f27df x1846780607021312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231763 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231738.139562:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607021312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231763 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231738.139576:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231738.139580:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231738.139585:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607021312:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231738.139594:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021376:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231738.139621:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231738/real 1761231738] req@0000000017386078 x1846780607021376/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231763 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231738.139633:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607021376/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231763 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231738.139642:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231738.139645:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231738.139647:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021376:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231738.139689:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021440:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231738.139719:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231738/real 1761231738] req@00000000c069b612 x1846780607021440/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231763 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231738.139740:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607021440/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231763 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231738.139750:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231738.139752:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231738.139755:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021440:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231738.139790:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021504:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231738.139815:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231738/real 1761231738] req@0000000094135f44 x1846780607021504/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231763 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231738.139871:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607021504/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231763 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231738.139883:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231738.139886:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231738.139889:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021504:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231738.587048:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231739.611162:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231740.635110:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231741.659169:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231742.683130:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231743.259051:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231743.259064:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231743.259069:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231743.259082:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4615 00000100:00080000:0.0:1761231743.259086:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231743.259097:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231743.259139:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607021568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231743.259180:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231743.259185:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231743.259187:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231743.259192:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4615 00000100:00080000:0.0:1761231743.259195:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231743.259199:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231743.259210:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231743.259224:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607021632/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231743.259235:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231743.259239:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231743.259240:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231743.259244:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4615 00000100:00080000:0.0:1761231743.259246:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231743.259249:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231743.259253:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231743.259263:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607021696/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231743.259273:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231743.259276:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231743.259278:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231743.259284:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4615 00000100:00080000:0.0:1761231743.259287:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231743.259291:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231743.259296:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231743.259307:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607021760/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231743.259356:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607021568:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231743.259509:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231743/real 1761231743] req@0000000099514377 x1846780607021568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231768 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231743.259532:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607021568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231768 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231743.259547:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231743.259552:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231743.259557:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607021568:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231743.259567:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021632:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231743.259634:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231743/real 1761231743] req@000000000243251f x1846780607021632/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231768 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231743.259649:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607021632/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231768 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231743.259658:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231743.259663:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231743.259666:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021632:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231743.259673:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021696:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231743.259701:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231743/real 1761231743] req@00000000271a9d8e x1846780607021696/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231768 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231743.259713:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607021696/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231768 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231743.259723:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231743.259725:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231743.259728:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021696:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231743.259734:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021760:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231743.259755:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231743/real 1761231743] req@0000000023ab3a3f x1846780607021760/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231768 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231743.259767:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607021760/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231768 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231743.259776:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231743.259778:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231743.259781:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021760:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231743.707169:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231744.731096:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231745.755185:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231746.779022:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231747.803164:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231748.379018:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231748.379029:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231748.379034:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231748.379047:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4620 00000100:00080000:3.0:1761231748.379051:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231748.379059:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231748.379098:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607021824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231748.379151:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231748.379158:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231748.379161:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231748.379168:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4620 00000100:00080000:3.0:1761231748.379172:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231748.379180:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231748.379194:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231748.379217:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607021824:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231748.379226:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607021888/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231748.379251:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231748.379257:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231748.379260:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231748.379267:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4620 00000100:00080000:3.0:1761231748.379272:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231748.379279:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231748.379288:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231748.379309:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607021952/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231748.379328:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231748.379335:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1761231748.379338:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231748/real 1761231748] req@00000000c7e4e432 x1846780607021824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231773 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231748.379341:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231748.379357:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4620 00000100:00080000:3.0:1761231748.379365:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231748.379373:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:0.0:1761231748.379379:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607021824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231773 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000008:00100000:3.0:1761231748.379381:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00080000:0.0:1761231748.379402:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1761231748.379407:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607022016/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231748.379411:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231748.379427:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607021824:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231748.379477:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021888:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231748.379619:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231748/real 1761231748] req@00000000f87d0618 x1846780607021888/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231773 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231748.379643:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607021888/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231773 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231748.379670:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231748.379674:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231748.379680:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021888:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231748.379692:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021952:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231748.379765:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231748/real 1761231748] req@000000007361665d x1846780607021952/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231773 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231748.379786:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607021952/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231773 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231748.379800:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231748.379804:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231748.379808:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607021952:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231748.379817:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022016:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231748.379857:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231748/real 1761231748] req@0000000031e043f4 x1846780607022016/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231773 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231748.379916:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607022016/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231773 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231748.379931:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231748.379934:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231748.379939:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022016:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231748.827077:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231749.851171:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231750.875099:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231751.899117:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231752.923003:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231753.499054:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231753.499067:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231753.499072:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231753.499086:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4625 00000100:00080000:2.0:1761231753.499090:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231753.499099:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231753.499145:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607022080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231753.499181:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231753.499186:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231753.499188:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231753.499193:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4625 00000100:00080000:2.0:1761231753.499196:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231753.499199:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231753.499210:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231753.499224:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607022144/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231753.499237:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231753.499242:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231753.499244:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231753.499249:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4625 00000100:00080000:2.0:1761231753.499251:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231753.499255:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231753.499261:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231753.499262:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607022080:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761231753.499278:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607022208/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231753.499292:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231753.499296:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231753.499298:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231753.499302:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4625 00000100:00080000:2.0:1761231753.499305:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231753.499308:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231753.499313:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231753.499326:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607022272/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231753.499383:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231753/real 1761231753] req@0000000064978143 x1846780607022080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231778 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231753.499406:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607022080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231778 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231753.499456:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231753.499461:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231753.499467:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607022080:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231753.499475:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022144:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231753.499507:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231753/real 1761231753] req@0000000091106a97 x1846780607022144/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231778 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231753.499521:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607022144/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231778 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231753.499529:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231753.499531:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231753.499534:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022144:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231753.499563:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022208:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231753.499587:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231753/real 1761231753] req@0000000094135f44 x1846780607022208/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231778 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231753.499607:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607022208/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231778 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231753.499617:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231753.499619:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231753.499621:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022208:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231753.499627:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022272:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231753.499647:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231753/real 1761231753] req@00000000c069b612 x1846780607022272/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231778 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231753.499659:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607022272/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231778 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231753.499668:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231753.499670:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231753.499673:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022272:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231753.947096:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231754.971185:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231755.995188:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231757.019137:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231758.043111:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231758.618955:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231758.618969:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231758.618974:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231758.618987:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4631 00000100:00080000:2.0:1761231758.618992:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231758.619000:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231758.619047:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607022336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231758.619096:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231758.619101:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231758.619103:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231758.619108:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4631 00000100:00080000:2.0:1761231758.619111:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231758.619115:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231758.619125:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231758.619139:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607022400/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231758.619150:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231758.619154:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231758.619156:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231758.619159:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4631 00000100:00080000:2.0:1761231758.619161:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231758.619166:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231758.619174:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231758.619210:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607022464/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231758.619219:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607022336:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761231758.619250:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231758.619259:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231758.619262:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231758.619273:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4631 00000100:00080000:2.0:1761231758.619278:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231758.619285:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231758.619294:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231758.619314:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607022528/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231758.619368:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231758/real 1761231758] req@0000000017386078 x1846780607022336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231783 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231758.619405:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607022336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231783 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231758.619426:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231758.619431:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231758.619456:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607022336:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231758.619482:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022400:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231758.619541:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231758/real 1761231758] req@00000000f38f27df x1846780607022400/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231783 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231758.619557:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607022400/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231783 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231758.619566:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231758.619569:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231758.619573:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022400:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231758.619604:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022464:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231758.619634:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231758/real 1761231758] req@00000000c069b612 x1846780607022464/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231783 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231758.619646:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607022464/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231783 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231758.619655:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231758.619657:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231758.619660:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022464:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231758.619666:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022528:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231758.619687:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231758/real 1761231758] req@0000000094135f44 x1846780607022528/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231783 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231758.619699:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607022528/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231783 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231758.619708:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231758.619710:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231758.619713:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022528:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231759.067166:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231760.091211:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231761.115140:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231762.139207:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231763.163174:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231763.739093:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231763.739111:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231763.739117:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231763.739135:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4636 00000100:00080000:0.0:1761231763.739140:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231763.739152:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231763.739214:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607022592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231763.739258:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231763.739263:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231763.739294:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231763.739301:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4636 00000100:00080000:0.0:1761231763.739304:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231763.739310:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231763.739325:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231763.739344:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607022656/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231763.739357:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231763.739361:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231763.739364:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231763.739368:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4636 00000100:00080000:0.0:1761231763.739371:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231763.739375:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231763.739380:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231763.739397:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607022720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231763.739399:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607022592:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761231763.739421:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231763.739426:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231763.739428:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231763.739432:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4636 00000100:00080000:0.0:1761231763.739435:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231763.739439:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231763.739443:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231763.739458:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607022784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231763.739590:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231763/real 1761231763] req@000000005a096598 x1846780607022592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231788 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231763.739625:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607022592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231788 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231763.739646:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231763.739651:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231763.739660:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607022592:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231763.739697:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022656:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231763.739765:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231763/real 1761231763] req@00000000275aeee5 x1846780607022656/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231788 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231763.739781:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607022656/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231788 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231763.739791:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231763.739795:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231763.739799:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022656:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231763.739880:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022720:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231763.739917:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231763/real 1761231763] req@000000002ef1f54f x1846780607022720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231788 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231763.739931:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607022720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231788 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231763.739941:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231763.739944:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231763.739947:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022720:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231763.739953:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022784:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231763.739974:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231763/real 1761231763] req@0000000023ab3a3f x1846780607022784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231788 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231763.739987:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607022784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231788 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231763.739996:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231763.739998:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231763.740000:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022784:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231764.187130:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231765.211100:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231766.235150:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231767.259162:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231768.283135:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231768.859077:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231768.859089:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231768.859095:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231768.859108:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4641 00000100:00080000:1.0:1761231768.859113:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231768.859122:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231768.859170:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607022848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231768.859228:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231768.859233:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231768.859235:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231768.859240:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4641 00000100:00080000:1.0:1761231768.859243:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231768.859247:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231768.859257:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231768.859271:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607022912/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231768.859281:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231768.859285:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231768.859287:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231768.859291:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4641 00000100:00080000:1.0:1761231768.859293:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231768.859296:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231768.859300:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231768.859310:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607022976/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231768.859321:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231768.859326:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231768.859328:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231768.859333:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4641 00000100:00080000:1.0:1761231768.859335:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231768.859339:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231768.859344:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231768.859355:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607022848:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761231768.859358:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607023040/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231768.859477:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231768/real 1761231768] req@00000000b72feabb x1846780607022848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231793 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231768.859508:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607022848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231793 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231768.859520:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231768.859524:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231768.859529:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607022848:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231768.859537:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022912:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231768.859563:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231768/real 1761231768] req@00000000659d78e3 x1846780607022912/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231793 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231768.859575:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607022912/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231793 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231768.859583:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231768.859586:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231768.859589:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022912:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231768.859595:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022976:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231768.859618:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231768/real 1761231768] req@00000000db10af3c x1846780607022976/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231793 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231768.859630:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607022976/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231793 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231768.859639:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231768.859641:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231768.859643:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607022976:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231768.859674:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023040:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231768.859698:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231768/real 1761231768] req@00000000d07717c4 x1846780607023040/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231793 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231768.859709:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607023040/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231793 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231768.859717:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231768.859719:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231768.859722:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023040:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231769.307141:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231770.331248:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231771.355132:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231772.379206:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231773.403184:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231773.978993:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231773.979002:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231773.979006:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231773.979014:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4646 00000100:00080000:0.0:1761231773.979017:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231773.979023:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231773.979054:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607023104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231773.979076:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231773.979079:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231773.979080:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231773.979083:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4646 00000100:00080000:0.0:1761231773.979084:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231773.979086:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231773.979092:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231773.979099:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607023168/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231773.979105:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231773.979107:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231773.979108:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231773.979111:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4646 00000100:00080000:0.0:1761231773.979133:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231773.979135:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231773.979139:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231773.979147:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607023232/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231773.979153:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231773.979156:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231773.979157:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231773.979159:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4646 00000100:00080000:0.0:1761231773.979160:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231773.979162:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231773.979164:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231773.979169:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607023296/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231773.979180:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607023104:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231773.979267:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231773/real 1761231773] req@00000000271a9d8e x1846780607023104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231798 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231773.979283:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607023104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231798 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231773.979292:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231773.979295:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231773.979299:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607023104:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231773.979306:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023168:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231773.979333:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231773/real 1761231773] req@000000000243251f x1846780607023168/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231798 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231773.979344:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607023168/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231798 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231773.979357:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231773.979360:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231773.979363:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023168:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231773.979369:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023232:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231773.979396:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231773/real 1761231773] req@0000000099514377 x1846780607023232/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231798 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231773.979407:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607023232/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231798 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231773.979415:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231773.979418:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231773.979421:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023232:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231773.979448:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023296:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231773.979473:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231773/real 1761231773] req@00000000b8f2be78 x1846780607023296/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231798 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231773.979480:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607023296/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231798 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231773.979485:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231773.979486:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231773.979488:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023296:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231774.427186:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231775.451162:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231776.475677:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231777.499107:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231778.523147:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231779.099347:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231779.099369:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231779.099377:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231779.099396:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4651 00000100:00080000:1.0:1761231779.099403:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231779.099416:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231779.099484:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607023360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231779.099546:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231779.099552:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231779.099554:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231779.099560:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4651 00000100:00080000:1.0:1761231779.099563:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231779.099568:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231779.099583:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231779.099603:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607023424/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231779.099617:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231779.099622:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231779.099624:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231779.099629:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4651 00000100:00080000:1.0:1761231779.099633:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231779.099655:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231779.099664:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231779.099685:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607023488/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231779.099702:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761231779.099722:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607023360:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761231779.099724:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231779.099726:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231779.099731:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4651 00000100:00080000:1.0:1761231779.099733:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231779.099737:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231779.099744:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231779.099758:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607023552/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231779.099907:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231779/real 1761231779] req@00000000d07717c4 x1846780607023360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231804 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231779.099935:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607023360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231804 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231779.099958:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231779.099963:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231779.099970:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607023360:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231779.099984:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023424:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231779.100030:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231779/real 1761231779] req@00000000db10af3c x1846780607023424/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231804 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231779.100075:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607023424/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231804 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231779.100086:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231779.100089:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231779.100093:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023424:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231779.100134:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023488:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231779.100174:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231779/real 1761231779] req@00000000659d78e3 x1846780607023488/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231804 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231779.100189:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607023488/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231804 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231779.100200:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231779.100203:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231779.100207:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023488:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231779.100215:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023552:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231779.100248:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231779/real 1761231779] req@00000000b72feabb x1846780607023552/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231804 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231779.100263:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607023552/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231804 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231779.100275:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231779.100278:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231779.100282:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023552:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231779.547206:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231780.571230:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231781.595231:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231782.619157:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231783.643225:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231784.219143:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231784.219157:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231784.219162:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231784.219175:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4656 00000100:00080000:3.0:1761231784.219180:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231784.219188:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231784.219232:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607023616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231784.219274:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231784.219278:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231784.219280:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231784.219285:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4656 00000100:00080000:3.0:1761231784.219288:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231784.219291:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231784.219302:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231784.219314:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607023680/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231784.219324:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231784.219328:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231784.219339:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231784.219343:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4656 00000100:00080000:3.0:1761231784.219346:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231784.219349:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231784.219354:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231784.219364:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607023744/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231784.219374:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231784.219379:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231784.219382:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231784.219387:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4656 00000100:00080000:3.0:1761231784.219389:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231784.219392:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231784.219396:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231784.219406:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607023808/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231784.219421:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607023616:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231784.219534:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231784/real 1761231784] req@000000001f8e688e x1846780607023616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231809 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231784.219555:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607023616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231809 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231784.219568:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231784.219572:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231784.219577:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607023616:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231784.219584:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023680:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231784.219610:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231784/real 1761231784] req@0000000090b40c66 x1846780607023680/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231809 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231784.219631:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607023680/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231809 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231784.219640:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231784.219642:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231784.219645:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023680:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231784.219651:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023744:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231784.219672:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231784/real 1761231784] req@0000000083f16a6b x1846780607023744/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231809 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231784.219683:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607023744/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231809 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231784.219692:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231784.219694:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231784.219697:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023744:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231784.219729:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023808:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231784.219753:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231784/real 1761231784] req@00000000a58ca11e x1846780607023808/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231809 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231784.219764:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607023808/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231809 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231784.219773:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231784.219776:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231784.219780:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023808:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231784.667209:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231785.691119:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231786.715207:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231787.739188:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231788.763136:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231789.339048:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231789.339061:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231789.339065:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231789.339077:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4661 00000100:00080000:0.0:1761231789.339081:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231789.339089:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231789.339130:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607023872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231789.339165:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231789.339169:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231789.339171:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231789.339175:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4661 00000100:00080000:0.0:1761231789.339177:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231789.339181:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231789.339191:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231789.339204:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607023936/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231789.339215:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761231789.339220:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607023872:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:0.0:1761231789.339222:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231789.339226:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231789.339236:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4661 00000100:00080000:0.0:1761231789.339238:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231789.339242:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231789.339246:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231789.339264:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607024000/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231789.339273:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231789.339276:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231789.339278:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231789.339282:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4661 00000100:00080000:0.0:1761231789.339284:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231789.339288:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231789.339291:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231789.339302:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607024064/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231789.339345:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023936:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231789.339373:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024000:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231789.339391:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024064:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231789.787241:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231790.811124:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231791.835218:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761231792.411611:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231789/real 1761231792] req@000000007f60e210 x1846780607023936/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231814 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231792.411651:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607023936/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231814 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231792.411669:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231792.411676:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231792.411683:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607023936:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231792.411699:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231789/real 1761231792] req@0000000001b2f813 x1846780607024000/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231814 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231792.411716:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607024000/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231814 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231792.411730:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231792.411735:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231792.411740:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024000:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231792.411748:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231789/real 1761231792] req@00000000b8f2be78 x1846780607024064/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231814 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231792.411764:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607024064/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231814 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231792.411776:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231792.411781:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231792.411785:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024064:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231792.411793:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231789/real 1761231792] req@00000000fcce404c x1846780607023872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231814 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231792.411810:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607023872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231814 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231792.411821:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231792.411892:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231792.411901:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607023872:192.168.201.104@tcp:250:kworker/u8:0.0 00010000:00010000:2.0:1761231792.859003:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231793.883029:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231794.458961:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231794.458971:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231794.458975:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231794.458985:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4666 00000100:00080000:0.0:1761231794.458989:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231794.458996:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231794.459035:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607024128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231794.459081:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231794.459085:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231794.459086:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231794.459090:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4666 00000100:00080000:0.0:1761231794.459092:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231794.459095:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231794.459103:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231794.459114:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607024192/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231794.459121:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231794.459136:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231794.459138:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231794.459159:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4666 00000100:00080000:0.0:1761231794.459162:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231794.459172:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1761231794.459176:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607024128:192.168.201.104@tcp:250:kworker/u8:0.0 00000008:00100000:0.0:1761231794.459182:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231794.459202:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607024256/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231794.459215:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231794.459221:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231794.459223:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231794.459227:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4666 00000100:00080000:0.0:1761231794.459230:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231794.459235:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231794.459240:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231794.459252:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607024320/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231794.459267:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231794/real 1761231794] req@0000000099514377 x1846780607024128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231819 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231794.459283:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607024128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231819 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231794.459293:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231794.459297:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231794.459300:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607024128:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231794.459306:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024192:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231794.459333:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231794/real 1761231794] req@000000000243251f x1846780607024192/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231819 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231794.459347:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607024192/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231819 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231794.459355:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231794.459358:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231794.459362:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024192:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231794.459390:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024256:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231794.459421:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231794/real 1761231794] req@00000000271a9d8e x1846780607024256/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231819 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231794.459436:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607024256/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231819 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231794.459446:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231794.459450:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231794.459453:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024256:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231794.459460:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024320:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231794.459491:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231794/real 1761231794] req@0000000023ab3a3f x1846780607024320/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231819 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231794.459504:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607024320/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231819 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231794.459513:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231794.459516:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231794.459519:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024320:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231794.907218:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231795.931194:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231796.955153:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231797.979172:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231799.002981:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231799.579022:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231799.579036:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231799.579046:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231799.579061:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4671 00000100:00080000:3.0:1761231799.579066:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231799.579074:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231799.579129:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607024384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231799.579161:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231799.579165:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231799.579167:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231799.579171:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4671 00000100:00080000:3.0:1761231799.579173:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231799.579178:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231799.579189:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231799.579201:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607024448/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231799.579210:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231799.579214:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1761231799.579216:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607024384:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761231799.579217:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231799.579226:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4671 00000100:00080000:3.0:1761231799.579229:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231799.579232:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231799.579237:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231799.579247:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607024512/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231799.579254:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231799.579257:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231799.579259:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231799.579262:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4671 00000100:00080000:3.0:1761231799.579264:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231799.579267:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231799.579270:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231799.579281:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607024576/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231799.579357:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231799/real 1761231799] req@000000002ce934d3 x1846780607024384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231824 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231799.579376:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607024384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231824 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231799.579388:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231799.579392:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231799.579395:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607024384:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231799.579426:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024448:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231799.579450:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231799/real 1761231799] req@000000006572b591 x1846780607024448/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231824 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231799.579459:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607024448/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231824 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231799.579465:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231799.579468:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231799.579470:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024448:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231799.579475:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024512:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231799.579492:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231799/real 1761231799] req@00000000fc01c41c x1846780607024512/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231824 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231799.579500:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607024512/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231824 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231799.579508:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231799.579510:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231799.579511:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024512:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231799.579515:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024576:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231799.579530:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231799/real 1761231799] req@00000000408f088c x1846780607024576/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231824 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231799.579537:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607024576/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231824 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231799.579543:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231799.579545:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231799.579547:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024576:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231800.027205:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231801.051066:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231802.075137:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231803.099184:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231804.123124:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231804.698957:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231804.698966:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231804.698969:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231804.698978:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4677 00000100:00080000:1.0:1761231804.698981:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231804.698987:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231804.699021:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607024640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231804.699042:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231804.699045:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231804.699046:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231804.699049:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4677 00000100:00080000:1.0:1761231804.699050:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231804.699052:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231804.699059:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231804.699066:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607024704/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231804.699072:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231804.699074:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231804.699076:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231804.699078:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4677 00000100:00080000:1.0:1761231804.699079:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231804.699081:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231804.699084:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231804.699090:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607024768/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231804.699095:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761231804.699098:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607024640:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761231804.699099:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231804.699101:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231804.699107:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4677 00000100:00080000:1.0:1761231804.699108:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231804.699110:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231804.699113:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231804.699120:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607024832/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231804.699186:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231804/real 1761231804] req@00000000b72feabb x1846780607024640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231829 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231804.699200:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607024640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231829 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231804.699207:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231804.699209:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231804.699212:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607024640:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231804.699217:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024704:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231804.699234:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231804/real 1761231804] req@00000000659d78e3 x1846780607024704/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231829 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231804.699240:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607024704/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231829 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231804.699244:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231804.699246:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231804.699247:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024704:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231804.699265:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024768:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231804.699278:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231804/real 1761231804] req@00000000db10af3c x1846780607024768/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231829 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231804.699284:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607024768/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231829 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231804.699293:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231804.699294:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231804.699295:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024768:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231804.699298:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024832:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231804.699307:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231804/real 1761231804] req@00000000d07717c4 x1846780607024832/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231829 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231804.699313:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607024832/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231829 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231804.699318:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231804.699319:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231804.699321:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024832:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231805.147117:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231806.171006:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231807.195129:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231808.219137:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231809.243109:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231809.819052:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231809.819069:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231809.819075:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231809.819093:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4682 00000100:00080000:0.0:1761231809.819099:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231809.819110:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231809.819169:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607024896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231809.819220:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231809.819226:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231809.819228:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231809.819234:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4682 00000100:00080000:0.0:1761231809.819236:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231809.819240:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231809.819253:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231809.819266:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607024960/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231809.819277:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231809.819281:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231809.819283:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231809.819287:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4682 00000100:00080000:0.0:1761231809.819289:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231809.819292:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231809.819298:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231809.819313:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607025024/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231809.819324:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231809.819327:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231809.819329:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231809.819333:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4682 00000100:00080000:0.0:1761231809.819336:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231809.819342:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231809.819348:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231809.819355:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607024896:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231809.819367:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607025088/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231809.819510:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231809/real 1761231809] req@000000002ef1f54f x1846780607024896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231834 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231809.819534:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607024896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231834 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231809.819548:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231809.819552:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231809.819557:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607024896:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231809.819565:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024960:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231809.819637:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231809/real 1761231809] req@00000000275aeee5 x1846780607024960/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231834 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231809.819651:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607024960/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231834 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231809.819661:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231809.819664:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231809.819667:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607024960:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231809.819674:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025024:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231809.819701:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231809/real 1761231809] req@000000005a096598 x1846780607025024/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231834 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231809.819713:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607025024/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231834 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231809.819723:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231809.819726:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231809.819729:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025024:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231809.819770:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025088:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231809.819795:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231809/real 1761231809] req@0000000023ab3a3f x1846780607025088/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231834 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231809.819807:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607025088/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231834 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231809.819816:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231809.819818:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231809.819821:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025088:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231810.267197:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231811.291174:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231812.315094:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231813.339149:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231814.363130:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231814.939114:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231814.939138:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231814.939143:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231814.939156:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4687 00000100:00080000:0.0:1761231814.939161:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231814.939170:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231814.939220:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607025152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231814.939255:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231814.939261:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231814.939264:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231814.939270:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4687 00000100:00080000:0.0:1761231814.939274:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231814.939279:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231814.939290:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231814.939307:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607025216/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761231814.939319:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607025152:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:0.0:1761231814.939346:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231814.939350:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231814.939352:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231814.939358:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4687 00000100:00080000:0.0:1761231814.939360:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231814.939364:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231814.939370:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231814.939386:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607025280/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231814.939395:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231814.939399:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231814.939400:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231814.939404:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4687 00000100:00080000:0.0:1761231814.939406:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231814.939410:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231814.939415:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231814.939429:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607025344/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761231814.939591:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231814/real 1761231814] req@00000000271a9d8e x1846780607025152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231839 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231814.939624:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607025152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231839 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231814.939643:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231814.939649:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231814.939656:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607025152:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231814.939694:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025216:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231814.939754:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231814/real 1761231814] req@000000000243251f x1846780607025216/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231839 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231814.939778:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607025216/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231839 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231814.939792:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231814.939798:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231814.939803:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025216:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231814.939875:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025280:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231814.939937:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231814/real 1761231814] req@0000000099514377 x1846780607025280/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231839 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231814.939959:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607025280/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231839 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231814.939976:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231814.939980:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231814.939986:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025280:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231814.939998:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025344:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231814.940046:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231814/real 1761231814] req@00000000fcce404c x1846780607025344/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231839 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231814.940067:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607025344/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231839 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231814.940081:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231814.940085:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231814.940090:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025344:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231815.387122:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231816.410994:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231817.435036:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231818.459054:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231819.483229:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231820.058933:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231820.058942:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231820.058946:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231820.058955:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4692 00000100:00080000:3.0:1761231820.058958:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231820.058964:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231820.058998:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607025408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231820.059021:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231820.059024:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231820.059025:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231820.059027:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4692 00000100:00080000:3.0:1761231820.059029:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231820.059032:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231820.059038:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231820.059047:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607025472/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231820.059054:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231820.059056:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231820.059057:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231820.059059:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4692 00000100:00080000:3.0:1761231820.059061:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231820.059062:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231820.059066:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231820.059069:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607025408:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231820.059076:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607025536/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231820.059083:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231820.059085:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231820.059086:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231820.059088:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4692 00000100:00080000:3.0:1761231820.059090:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231820.059091:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231820.059094:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231820.059101:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607025600/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761231820.059147:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231820/real 1761231820] req@00000000408f088c x1846780607025408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231845 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231820.059161:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607025408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231845 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231820.059172:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231820.059175:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231820.059179:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607025408:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231820.059186:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025472:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231820.059215:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231820/real 1761231820] req@00000000fc01c41c x1846780607025472/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231845 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231820.059226:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607025472/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231845 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231820.059233:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231820.059235:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231820.059238:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025472:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231820.059264:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025536:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231820.059292:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231820/real 1761231820] req@000000006572b591 x1846780607025536/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231845 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231820.059304:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607025536/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231845 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231820.059312:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231820.059315:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231820.059318:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025536:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231820.059324:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025600:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231820.059352:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231820/real 1761231820] req@000000002ce934d3 x1846780607025600/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231845 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231820.059363:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607025600/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231845 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231820.059372:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231820.059374:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231820.059378:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025600:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231820.507134:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231821.531150:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231822.555083:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231823.579037:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231824.603154:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231825.178905:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231825.178917:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231825.178921:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231825.178932:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4697 00000100:00080000:3.0:1761231825.178935:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231825.178942:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231825.178977:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607025664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231825.179001:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231825.179005:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231825.179006:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231825.179010:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4697 00000100:00080000:3.0:1761231825.179012:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231825.179014:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231825.179023:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231825.179034:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607025728/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231825.179044:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231825.179047:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231825.179048:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231825.179055:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4697 00000100:00100000:1.0:1761231825.179056:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607025664:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761231825.179058:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231825.179064:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231825.179071:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231825.179083:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607025792/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231825.179090:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231825.179094:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231825.179096:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231825.179099:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4697 00000100:00080000:3.0:1761231825.179101:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231825.179104:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231825.179107:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231825.179115:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607025856/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761231825.179170:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231825/real 1761231825] req@00000000a58ca11e x1846780607025664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231850 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231825.179187:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607025664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231850 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231825.179196:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231825.179199:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231825.179203:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607025664:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231825.179226:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025728:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231825.179250:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231825/real 1761231825] req@0000000083f16a6b x1846780607025728/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231850 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231825.179259:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607025728/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231850 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231825.179265:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231825.179268:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231825.179270:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025728:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231825.179275:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025792:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231825.179291:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231825/real 1761231825] req@0000000090b40c66 x1846780607025792/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231850 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231825.179300:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607025792/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231850 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231825.179307:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231825.179309:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231825.179311:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025792:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231825.179314:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025856:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231825.179329:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231825/real 1761231825] req@000000001f8e688e x1846780607025856/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231850 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231825.179337:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607025856/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231850 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231825.179344:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231825.179345:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231825.179369:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025856:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231825.627169:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231826.651097:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231827.675157:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231828.699169:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231829.723083:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231830.298956:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231830.298966:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231830.298970:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231830.298980:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4702 00000100:00080000:3.0:1761231830.298982:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231830.298988:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231830.299025:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607025920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231830.299047:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231830.299050:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231830.299052:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231830.299056:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4702 00000100:00080000:3.0:1761231830.299058:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231830.299060:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231830.299067:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231830.299076:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607025984/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231830.299084:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231830.299086:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231830.299087:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231830.299090:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4702 00000100:00080000:3.0:1761231830.299091:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231830.299093:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231830.299096:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231830.299104:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607026048/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231830.299109:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231830.299112:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231830.299113:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231830.299116:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4702 00000100:00080000:3.0:1761231830.299117:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231830.299120:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231830.299132:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231830.299140:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607025920:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231830.299147:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607026112/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761231830.299254:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231830/real 1761231830] req@0000000031e043f4 x1846780607025920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231855 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231830.299349:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607025920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231855 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231830.299357:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231830.299360:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231830.299364:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607025920:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231830.299370:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025984:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231830.299391:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231830/real 1761231830] req@000000007361665d x1846780607025984/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231855 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231830.299399:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607025984/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231855 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231830.299404:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231830.299406:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231830.299408:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607025984:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231830.299411:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026048:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231830.299424:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231830/real 1761231830] req@00000000f87d0618 x1846780607026048/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231855 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231830.299431:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607026048/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231855 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231830.299437:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231830.299438:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231830.299440:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026048:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231830.299463:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026112:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231830.299476:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231830/real 1761231830] req@00000000c7e4e432 x1846780607026112/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231855 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231830.299483:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607026112/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231855 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231830.299489:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231830.299490:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231830.299492:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026112:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231830.747079:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231831.770957:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231832.795184:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231833.819040:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231834.842980:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231835.418949:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231835.418960:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231835.418964:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231835.418975:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4707 00000100:00080000:3.0:1761231835.418979:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231835.418987:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231835.419024:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607026176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231835.419052:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231835.419055:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231835.419057:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231835.419060:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4707 00000100:00080000:3.0:1761231835.419062:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231835.419065:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231835.419073:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231835.419082:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607026240/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231835.419089:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231835.419092:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231835.419093:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231835.419096:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4707 00000100:00080000:3.0:1761231835.419098:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231835.419100:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231835.419120:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231835.419131:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607026304/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231835.419139:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231835.419142:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231835.419143:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231835.419148:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4707 00000100:00080000:3.0:1761231835.419150:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231835.419154:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231835.419158:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231835.419167:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607026368/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761231835.419177:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607026176:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231835.419334:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231835/real 1761231835] req@00000000bd684ccc x1846780607026176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231860 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231835.419351:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607026176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231860 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231835.419361:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231835.419364:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231835.419367:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607026176:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231835.419374:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026240:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231835.419399:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231835/real 1761231835] req@0000000018a8f5da x1846780607026240/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231860 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231835.419408:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607026240/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231860 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231835.419422:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231835.419424:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231835.419427:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026240:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231835.419431:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026304:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231835.419448:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231835/real 1761231835] req@000000005a9bdefd x1846780607026304/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231860 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231835.419457:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607026304/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231860 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231835.419465:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231835.419467:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231835.419469:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026304:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231835.419494:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026368:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231835.419512:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231835/real 1761231835] req@0000000086ccb527 x1846780607026368/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231860 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231835.419520:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607026368/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231860 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231835.419527:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231835.419529:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231835.419531:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026368:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231835.867025:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231836.891137:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231837.915070:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231838.939110:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231839.963006:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231840.539041:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231840.539054:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231840.539060:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231840.539072:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4712 00000100:00080000:2.0:1761231840.539076:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231840.539084:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231840.539192:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607026432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231840.539229:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231840.539233:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231840.539236:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231840.539241:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4712 00000100:00080000:2.0:1761231840.539243:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231840.539247:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231840.539260:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231840.539274:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607026496/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231840.539284:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231840.539288:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231840.539290:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231840.539293:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4712 00000100:00080000:2.0:1761231840.539295:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231840.539298:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231840.539303:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231840.539316:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607026560/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231840.539328:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231840.539332:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231840.539334:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231840.539338:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4712 00000100:00080000:2.0:1761231840.539341:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231840.539345:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231840.539351:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231840.539358:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607026432:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761231840.539366:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607026624/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761231840.539485:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231840/real 1761231840] req@0000000091106a97 x1846780607026432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231865 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761231840.539506:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607026432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231865 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761231840.539519:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231840.539523:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231840.539528:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607026432:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761231840.539536:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026496:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761231840.539562:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231840/real 1761231840] req@0000000064978143 x1846780607026496/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231865 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761231840.539575:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607026496/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231865 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761231840.539583:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231840.539586:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231840.539588:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026496:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761231840.539594:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026560:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761231840.539615:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231840/real 1761231840] req@0000000094135f44 x1846780607026560/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231865 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761231840.539626:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607026560/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231865 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761231840.539635:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231840.539638:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231840.539641:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026560:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761231840.539670:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026624:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761231840.539694:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231840/real 1761231840] req@00000000c069b612 x1846780607026624/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231865 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761231840.539706:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607026624/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231865 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761231840.539715:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231840.539717:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231840.539720:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026624:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231840.987093:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231842.011159:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231843.035178:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231844.059140:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231845.083182:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231845.659097:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231845.659111:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231845.659117:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231845.659130:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4718 00000100:00080000:0.0:1761231845.659135:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231845.659144:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231845.659191:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607026688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231845.659228:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231845.659233:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231845.659235:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231845.659240:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4718 00000100:00080000:0.0:1761231845.659243:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231845.659246:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231845.659255:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231845.659273:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607026752/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231845.659285:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231845.659290:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231845.659292:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231845.659297:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4718 00000100:00080000:0.0:1761231845.659299:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231845.659303:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231845.659308:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231845.659324:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607026688:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231845.659325:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607026816/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231845.659348:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231845.659352:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231845.659354:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231845.659357:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4718 00000100:00080000:0.0:1761231845.659359:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231845.659363:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231845.659368:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231845.659381:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607026880/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761231845.659452:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231845/real 1761231845] req@00000000b8f2be78 x1846780607026688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231870 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231845.659474:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607026688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231870 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231845.659486:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231845.659490:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231845.659494:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607026688:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231845.659503:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026752:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231845.659531:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231845/real 1761231845] req@0000000001b2f813 x1846780607026752/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231870 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231845.659553:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607026752/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231870 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231845.659562:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231845.659564:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231845.659567:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026752:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231845.659596:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026816:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231845.659620:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231845/real 1761231845] req@000000007f60e210 x1846780607026816/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231870 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231845.659632:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607026816/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231870 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231845.659641:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231845.659643:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231845.659646:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026816:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231845.659651:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026880:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231845.659672:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231845/real 1761231845] req@00000000fcce404c x1846780607026880/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231870 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231845.659683:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607026880/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231870 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231845.659693:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231845.659695:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231845.659698:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607026880:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231846.107137:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231847.131177:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231848.155229:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231849.179163:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231850.203159:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231850.779091:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231850.779109:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231850.779115:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231850.779131:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4723 00000100:00080000:3.0:1761231850.779138:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231850.779149:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231850.779199:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607026944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231850.779239:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231850.779243:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231850.779245:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231850.779250:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4723 00000100:00080000:3.0:1761231850.779253:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231850.779257:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231850.779268:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231850.779281:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607027008/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231850.779292:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231850.779297:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231850.779298:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231850.779302:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4723 00000100:00080000:3.0:1761231850.779305:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231850.779308:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231850.779312:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231850.779325:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607027072/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231850.779335:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231850.779338:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231850.779340:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231850.779346:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4723 00000100:00080000:3.0:1761231850.779350:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231850.779353:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231850.779358:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231850.779368:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607027136/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761231850.779377:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607026944:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231850.779489:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231850/real 1761231850] req@000000004e3f612a x1846780607026944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231875 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231850.779510:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607026944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231875 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231850.779522:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231850.779526:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231850.779530:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607026944:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231850.779539:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027008:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231850.779564:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231850/real 1761231850] req@00000000daa39bb1 x1846780607027008/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231875 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231850.779576:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607027008/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231875 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231850.779584:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231850.779587:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231850.779590:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027008:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231850.779596:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027072:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231850.779617:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231850/real 1761231850] req@000000003ad149cc x1846780607027072/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231875 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231850.779628:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607027072/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231875 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231850.779638:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231850.779640:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231850.779643:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027072:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231850.779675:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027136:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231850.779697:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231850/real 1761231850] req@0000000088473b3e x1846780607027136/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231875 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231850.779708:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607027136/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231875 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231850.779717:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231850.779719:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231850.779722:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027136:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231851.227189:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231852.251164:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231853.275180:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231854.299022:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231855.323134:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231855.898999:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231855.899012:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231855.899017:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231855.899030:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4728 00000100:00080000:3.0:1761231855.899035:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231855.899043:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231855.899090:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607027200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231855.899126:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231855.899130:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231855.899132:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231855.899137:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4728 00000100:00080000:3.0:1761231855.899139:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231855.899143:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231855.899152:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231855.899164:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607027264/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231855.899174:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231855.899178:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231855.899179:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231855.899183:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4728 00000100:00080000:3.0:1761231855.899185:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231855.899188:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231855.899192:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231855.899201:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607027328/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231855.899210:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231855.899214:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231855.899215:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231855.899219:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4728 00000100:00080000:3.0:1761231855.899221:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231855.899227:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231855.899231:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231855.899242:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607027392/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761231855.899266:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607027200:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231855.899386:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027264:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231855.899430:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027328:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231855.899481:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027392:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231856.347026:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231857.371098:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231858.395159:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761231858.971484:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231855/real 1761231858] req@000000004e3f612a x1846780607027392/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231880 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231858.971533:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607027392/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231880 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231858.971565:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231858.971571:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231858.971579:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027392:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231858.971590:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231855/real 1761231858] req@0000000088473b3e x1846780607027200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231880 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231858.971658:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607027200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231880 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231858.971674:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231858.971678:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231858.971683:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607027200:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231858.971695:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231855/real 1761231858] req@000000003ad149cc x1846780607027264/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231880 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231858.971715:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607027264/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231880 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231858.971739:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231858.971744:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231858.971749:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027264:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231858.971759:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231855/real 1761231858] req@00000000daa39bb1 x1846780607027328/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231880 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231858.971777:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607027328/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231880 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231858.971802:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231858.971806:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231858.971811:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027328:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:2.0:1761231859.419039:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231860.443121:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231861.018966:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231861.018980:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231861.018984:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231861.018999:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4733 00000100:00080000:3.0:1761231861.019004:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231861.019013:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231861.019056:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607027456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231861.019091:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231861.019095:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231861.019097:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231861.019102:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4733 00000100:00080000:3.0:1761231861.019105:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231861.019108:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231861.019118:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231861.019130:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607027520/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231861.019140:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231861.019144:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231861.019145:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231861.019151:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4733 00000100:00080000:3.0:1761231861.019153:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231861.019157:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231861.019162:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231861.019173:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607027584/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231861.019182:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231861.019186:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231861.019189:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231861.019193:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4733 00000100:00080000:3.0:1761231861.019195:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231861.019200:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231861.019205:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231861.019214:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607027648/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761231861.019230:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607027456:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231861.019348:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231861/real 1761231861] req@0000000086ccb527 x1846780607027456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231886 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231861.019377:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607027456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231886 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231861.019393:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231861.019399:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231861.019405:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607027456:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231861.019417:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027520:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231861.019469:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231861/real 1761231861] req@000000005a9bdefd x1846780607027520/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231886 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231861.019490:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607027520/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231886 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231861.019504:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231861.019509:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231861.019514:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027520:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231861.019525:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027584:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231861.019595:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231861/real 1761231861] req@0000000018a8f5da x1846780607027584/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231886 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231861.019618:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607027584/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231886 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231861.019644:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231861.019646:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231861.019651:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027584:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231861.019687:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027648:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231861.019723:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231861/real 1761231861] req@00000000bd684ccc x1846780607027648/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231886 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231861.019736:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607027648/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231886 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231861.019745:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231861.019747:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231861.019750:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027648:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231861.467127:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231862.491096:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231863.515054:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231864.539131:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231865.563181:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231866.138994:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231866.139008:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231866.139013:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231866.139026:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4738 00000100:00080000:3.0:1761231866.139031:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231866.139039:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231866.139086:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607027712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231866.139120:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231866.139126:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231866.139128:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231866.139133:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4738 00000100:00080000:3.0:1761231866.139135:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231866.139139:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231866.139148:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231866.139160:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607027776/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231866.139171:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231866.139174:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231866.139176:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231866.139180:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4738 00000100:00080000:3.0:1761231866.139182:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231866.139185:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231866.139189:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231866.139200:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607027840/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231866.139209:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231866.139213:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231866.139215:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231866.139220:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4738 00000100:00080000:3.0:1761231866.139223:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231866.139227:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231866.139232:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231866.139244:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607027904/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761231866.139266:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607027712:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231866.139383:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231866/real 1761231866] req@00000000c7e4e432 x1846780607027712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231891 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231866.139405:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607027712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231891 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231866.139417:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231866.139422:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231866.139426:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607027712:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761231866.139435:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027776:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231866.139461:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231866/real 1761231866] req@00000000f87d0618 x1846780607027776/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231891 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231866.139474:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607027776/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231891 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231866.139482:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231866.139484:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231866.139487:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027776:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761231866.139493:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027840:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231866.139514:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231866/real 1761231866] req@000000007361665d x1846780607027840/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231891 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231866.139526:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607027840/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231891 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231866.139535:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231866.139537:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231866.139539:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027840:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231866.139572:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027904:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761231866.139595:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231866/real 1761231866] req@0000000031e043f4 x1846780607027904/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231891 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761231866.139615:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607027904/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231891 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761231866.139624:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231866.139626:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231866.139629:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607027904:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231866.587168:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231867.611087:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231868.634974:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231869.659153:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231870.683096:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231871.258982:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231871.258992:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231871.258995:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231871.259005:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4743 00000100:00080000:2.0:1761231871.259008:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231871.259014:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231871.259049:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607027968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231871.259076:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231871.259079:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231871.259081:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231871.259084:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4743 00000100:00080000:2.0:1761231871.259086:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231871.259088:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231871.259095:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231871.259103:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607028032/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231871.259110:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231871.259114:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231871.259115:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231871.259118:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4743 00000100:00080000:2.0:1761231871.259120:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231871.259122:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231871.259125:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231871.259133:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607028096/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231871.259138:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231871.259140:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231871.259141:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231871.259144:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4743 00000100:00080000:2.0:1761231871.259146:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231871.259149:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231871.259152:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231871.259160:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607028160/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761231871.259178:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607027968:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761231871.259264:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231871/real 1761231871] req@00000000f38f27df x1846780607027968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231896 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761231871.259278:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607027968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231896 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761231871.259286:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231871.259289:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231871.259292:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607027968:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761231871.259297:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028032:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761231871.259315:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231871/real 1761231871] req@0000000017386078 x1846780607028032/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231896 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761231871.259323:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607028032/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231896 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761231871.259328:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231871.259329:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231871.259331:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028032:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761231871.259336:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028096:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761231871.259349:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231871/real 1761231871] req@00000000c069b612 x1846780607028096/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231896 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761231871.259356:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607028096/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231896 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761231871.259362:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231871.259364:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231871.259366:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028096:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761231871.259369:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028160:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761231871.259382:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231871/real 1761231871] req@0000000094135f44 x1846780607028160/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231896 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761231871.259389:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607028160/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231896 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761231871.259395:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761231871.259397:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761231871.259398:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028160:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231871.707158:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231872.731120:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231873.755103:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231874.779032:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231875.803134:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231876.379043:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231876.379060:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231876.379066:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231876.379082:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4748 00000100:00080000:1.0:1761231876.379089:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231876.379100:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231876.379169:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607028224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231876.379224:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231876.379232:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231876.379235:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231876.379242:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4748 00000100:00080000:1.0:1761231876.379246:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231876.379252:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231876.379266:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231876.379290:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607028288/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231876.379309:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231876.379316:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231876.379319:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231876.379326:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4748 00000100:00080000:1.0:1761231876.379331:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231876.379338:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231876.379347:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231876.379371:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607028352/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231876.379391:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761231876.379400:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607028224:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:1.0:1761231876.379404:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231876.379412:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231876.379428:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4748 00000100:00080000:1.0:1761231876.379433:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231876.379440:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231876.379451:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231876.379480:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607028416/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231876.379578:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231876/real 1761231876] req@00000000d07717c4 x1846780607028224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231901 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231876.379659:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607028224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231901 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231876.379672:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231876.379676:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231876.379681:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607028224:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231876.379690:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028288:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231876.379721:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231876/real 1761231876] req@00000000db10af3c x1846780607028288/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231901 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231876.379732:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607028288/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231901 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231876.379739:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231876.379742:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231876.379745:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028288:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231876.379777:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028352:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231876.379799:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231876/real 1761231876] req@00000000659d78e3 x1846780607028352/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231901 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231876.379816:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607028352/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231901 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231876.379824:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231876.379879:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231876.379882:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028352:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231876.379889:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028416:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231876.379914:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231876/real 1761231876] req@00000000b72feabb x1846780607028416/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231901 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231876.379925:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607028416/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231901 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231876.379933:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231876.379935:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231876.379938:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028416:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231876.827025:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231877.851012:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231878.875104:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231879.899117:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231880.923031:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231881.499101:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231881.499115:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231881.499121:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231881.499134:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4753 00000100:00080000:3.0:1761231881.499138:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231881.499147:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231881.499191:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607028480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231881.499242:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231881.499256:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231881.499259:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231881.499269:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4753 00000100:00080000:3.0:1761231881.499276:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231881.499286:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231881.499302:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231881.499331:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607028544/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231881.499341:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607028480:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761231881.499370:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231881.499374:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231881.499375:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231881.499381:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4753 00000100:00080000:3.0:1761231881.499384:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231881.499388:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231881.499394:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231881.499415:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607028608/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231881.499449:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231881.499459:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231881.499463:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231881.499475:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4753 00000100:00080000:3.0:1761231881.499481:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1761231881.499487:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231881/real 1761231881] req@000000001f8e688e x1846780607028480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231906 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231881.499494:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231881.499542:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231881.499563:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607028672/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231881.499575:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607028480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231906 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231881.499611:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231881.499617:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231881.499626:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607028480:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231881.499662:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028544:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231881.499723:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231881/real 1761231881] req@0000000090b40c66 x1846780607028544/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231906 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231881.499756:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607028544/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231906 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231881.499771:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231881.499776:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231881.499782:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028544:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231881.499794:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028608:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231881.499903:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231881/real 1761231881] req@0000000083f16a6b x1846780607028608/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231906 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231881.499944:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607028608/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231906 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231881.499964:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231881.499970:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231881.499978:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028608:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231881.499992:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028672:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231881.500045:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231881/real 1761231881] req@00000000a58ca11e x1846780607028672/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231906 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231881.500067:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607028672/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231906 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231881.500083:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231881.500088:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231881.500094:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028672:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231881.947131:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231882.971085:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231883.995136:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231885.019163:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231886.043142:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231886.618939:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231886.618954:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231886.618959:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231886.618972:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4759 00000100:00080000:3.0:1761231886.618976:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231886.618985:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231886.619033:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607028736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231886.619068:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231886.619073:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231886.619075:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231886.619079:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4759 00000100:00080000:3.0:1761231886.619082:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231886.619086:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231886.619095:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231886.619108:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607028800/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231886.619118:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231886.619121:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231886.619122:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231886.619126:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4759 00000100:00080000:3.0:1761231886.619129:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231886.619132:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231886.619138:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231886.619152:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607028864/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231886.619162:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231886.619166:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231886.619168:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231886.619172:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4759 00000100:00080000:3.0:1761231886.619174:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231886.619178:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231886.619182:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231886.619192:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607028736:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231886.619196:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607028928/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231886.619320:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231886/real 1761231886] req@000000002ce934d3 x1846780607028736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231911 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231886.619337:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607028736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231911 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231886.619346:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231886.619349:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231886.619353:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607028736:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231886.619359:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028800:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231886.619379:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231886/real 1761231886] req@000000006572b591 x1846780607028800/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231911 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231886.619388:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607028800/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231911 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231886.619394:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231886.619395:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231886.619397:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028800:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231886.619401:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028864:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231886.619417:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231886/real 1761231886] req@00000000fc01c41c x1846780607028864/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231911 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231886.619426:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607028864/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231911 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231886.619452:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231886.619454:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231886.619457:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028864:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231886.619483:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028928:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231886.619503:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231886/real 1761231886] req@00000000408f088c x1846780607028928/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231911 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231886.619513:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607028928/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231911 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231886.619519:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231886.619521:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231886.619523:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607028928:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231887.067145:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231888.091170:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231889.115050:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231890.139120:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231891.163032:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231891.738902:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231891.738913:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231891.738917:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231891.738927:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4764 00000100:00080000:3.0:1761231891.738931:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231891.738937:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231891.738989:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607028992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231891.739018:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231891.739023:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231891.739025:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231891.739031:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4764 00000100:00080000:3.0:1761231891.739035:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231891.739039:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231891.739051:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231891.739074:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607028992:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231891.739084:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607029056/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231891.739097:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231891.739102:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231891.739103:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231891.739109:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4764 00000100:00080000:3.0:1761231891.739111:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231891.739115:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231891.739121:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231891.739134:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607029120/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231891.739145:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231891.739148:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231891.739151:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231891.739155:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4764 00000100:00080000:3.0:1761231891.739157:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231891.739161:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231891.739166:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231891.739177:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231891/real 1761231891] req@00000000408f088c x1846780607028992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231916 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231891.739177:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607029184/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231891.739201:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607028992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231916 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231891.739215:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231891.739219:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231891.739225:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607028992:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231891.739253:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029056:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231891.739286:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231891/real 1761231891] req@00000000fc01c41c x1846780607029056/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231916 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231891.739296:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607029056/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231916 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231891.739303:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231891.739305:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231891.739308:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029056:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231891.739313:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029120:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231891.739330:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231891/real 1761231891] req@000000006572b591 x1846780607029120/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231916 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231891.739359:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607029120/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231916 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231891.739389:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231891.739398:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231891.739404:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029120:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231891.739412:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029184:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231891.739453:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231891/real 1761231891] req@000000002ce934d3 x1846780607029184/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231916 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231891.739465:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607029184/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231916 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231891.739473:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231891.739475:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231891.739478:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029184:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231892.187096:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231893.211103:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231894.235080:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231895.259143:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231896.283168:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231896.858943:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231896.858955:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231896.858959:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231896.858970:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4769 00000100:00080000:3.0:1761231896.858973:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231896.858981:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231896.859017:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607029248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231896.859046:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231896.859050:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231896.859052:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231896.859056:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4769 00000100:00080000:3.0:1761231896.859058:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231896.859062:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231896.859071:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231896.859082:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607029312/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231896.859090:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231896.859094:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231896.859095:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231896.859099:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4769 00000100:00080000:3.0:1761231896.859102:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231896.859106:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231896.859112:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231896.859125:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607029376/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231896.859128:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607029248:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761231896.859142:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231896.859146:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231896.859148:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231896.859152:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4769 00000100:00080000:3.0:1761231896.859155:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231896.859158:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231896.859163:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231896.859175:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607029440/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231896.859209:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231896/real 1761231896] req@00000000a58ca11e x1846780607029248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231921 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231896.859223:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607029248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231921 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231896.859230:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231896.859232:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231896.859236:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607029248:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231896.859241:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029312:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231896.859257:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231896/real 1761231896] req@0000000083f16a6b x1846780607029312/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231921 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231896.859269:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607029312/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231921 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231896.859273:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231896.859275:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231896.859276:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029312:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231896.859295:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029376:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231896.859324:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231896/real 1761231896] req@0000000090b40c66 x1846780607029376/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231921 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231896.859332:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607029376/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231921 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231896.859338:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231896.859339:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231896.859341:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029376:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231896.859344:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029440:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231896.859358:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231896/real 1761231896] req@000000001f8e688e x1846780607029440/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231921 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231896.859364:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607029440/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231921 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231896.859369:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231896.859370:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231896.859372:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029440:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231897.307168:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231898.331167:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231899.355129:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231900.379191:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231901.403184:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231901.978925:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231901.978936:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231901.978941:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231901.978953:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4774 00000100:00080000:3.0:1761231901.978957:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231901.978964:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231901.979005:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607029504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231901.979032:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231901.979035:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231901.979037:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231901.979041:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4774 00000100:00080000:3.0:1761231901.979043:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231901.979046:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231901.979057:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231901.979068:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607029568/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231901.979078:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761231901.979085:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607029504:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761231901.979085:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231901.979088:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231901.979098:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4774 00000100:00080000:3.0:1761231901.979100:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231901.979103:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231901.979107:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231901.979120:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607029632/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231901.979128:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231901.979132:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231901.979133:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231901.979136:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4774 00000100:00080000:3.0:1761231901.979138:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231901.979141:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231901.979145:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231901.979170:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607029696/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231901.979205:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231901/real 1761231901] req@0000000031e043f4 x1846780607029504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231926 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231901.979222:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607029504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231926 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231901.979232:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231901.979234:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231901.979238:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607029504:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231901.979284:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029568:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231901.979320:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231901/real 1761231901] req@000000007361665d x1846780607029568/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231926 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231901.979335:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607029568/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231926 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231901.979345:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231901.979349:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231901.979353:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029568:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231901.979361:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029632:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231901.979393:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231901/real 1761231901] req@00000000f87d0618 x1846780607029632/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231926 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231901.979408:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607029632/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231926 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231901.979419:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231901.979423:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231901.979426:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029632:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231901.979433:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029696:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231901.979456:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231901/real 1761231901] req@00000000c7e4e432 x1846780607029696/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231926 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231901.979465:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607029696/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231926 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231901.979471:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231901.979473:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231901.979475:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029696:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231902.427184:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231903.451107:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231904.475082:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231905.499052:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231906.523043:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231907.098960:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231907.098973:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231907.098978:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231907.098992:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4779 00000100:00080000:3.0:1761231907.098996:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231907.099005:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231907.099054:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607029760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231907.099088:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231907.099093:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231907.099095:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231907.099099:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4779 00000100:00080000:3.0:1761231907.099102:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231907.099105:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231907.099114:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231907.099127:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607029824/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231907.099137:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231907.099141:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231907.099142:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231907.099163:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4779 00000100:00080000:3.0:1761231907.099165:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231907.099168:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231907.099172:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231907.099184:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607029888/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231907.099228:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231907.099233:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231907.099235:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231907.099239:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4779 00000100:00080000:3.0:1761231907.099242:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231907.099246:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231907.099251:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231907.099262:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607029952/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231907.099282:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607029760:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231907.099395:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231907/real 1761231907] req@00000000bd684ccc x1846780607029760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231932 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231907.099425:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607029760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231932 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231907.099436:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231907.099440:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231907.099444:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607029760:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231907.099452:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029824:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231907.099477:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231907/real 1761231907] req@0000000018a8f5da x1846780607029824/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231932 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231907.099507:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607029824/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231932 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231907.099515:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231907.099517:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231907.099520:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029824:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231907.099526:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029888:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231907.099552:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231907/real 1761231907] req@000000005a9bdefd x1846780607029888/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231932 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231907.099564:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607029888/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231932 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231907.099574:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231907.099576:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231907.099579:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029888:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231907.099612:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029952:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231907.099635:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231907/real 1761231907] req@0000000086ccb527 x1846780607029952/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231932 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231907.099647:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607029952/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231932 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231907.099656:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231907.099658:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231907.099661:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607029952:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231907.547129:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231908.571221:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231909.595087:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231910.619135:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231911.643116:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231912.219073:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231912.219107:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231912.219113:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231912.219131:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4784 00000100:00080000:0.0:1761231912.219195:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231912.219205:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231912.219265:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607030016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231912.219309:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231912.219313:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231912.219315:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231912.219321:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4784 00000100:00080000:0.0:1761231912.219324:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231912.219328:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231912.219340:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231912.219352:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607030080/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231912.219361:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231912.219365:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231912.219366:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231912.219371:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4784 00000100:00080000:0.0:1761231912.219372:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231912.219375:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231912.219380:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231912.219390:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607030144/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231912.219398:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231912.219402:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231912.219403:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231912.219407:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4784 00000100:00080000:0.0:1761231912.219409:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231912.219413:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231912.219418:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231912.219428:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607030208/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231912.219456:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607030016:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231912.219583:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231912/real 1761231912] req@0000000099514377 x1846780607030016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231937 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231912.219604:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607030016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231937 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231912.219618:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231912.219622:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231912.219627:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607030016:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231912.219636:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030080:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231912.219663:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231912/real 1761231912] req@000000000243251f x1846780607030080/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231937 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231912.219676:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607030080/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231937 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231912.219684:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231912.219688:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231912.219692:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030080:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231912.219698:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030144:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231912.219723:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231912/real 1761231912] req@00000000271a9d8e x1846780607030144/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231937 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231912.219734:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607030144/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231937 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231912.219743:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231912.219745:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231912.219748:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030144:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231912.219782:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030208:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231912.219806:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231912/real 1761231912] req@0000000023ab3a3f x1846780607030208/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231937 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231912.219818:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607030208/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231937 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231912.219862:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231912.219866:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231912.219870:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030208:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231912.667220:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231913.691245:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231914.715100:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231915.739130:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231916.763175:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231917.339107:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231917.339126:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231917.339133:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231917.339150:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4789 00000100:00080000:3.0:1761231917.339156:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231917.339167:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231917.339232:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607030272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231917.339284:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231917.339290:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231917.339292:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231917.339297:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4789 00000100:00080000:3.0:1761231917.339300:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231917.339303:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231917.339317:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231917.339330:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607030336/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231917.339341:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231917.339345:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231917.339346:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231917.339351:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4789 00000100:00080000:3.0:1761231917.339353:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231917.339357:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231917.339377:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231917.339399:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607030400/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231917.339420:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607030272:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761231917.339437:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231917.339448:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231917.339451:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231917.339463:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4789 00000100:00080000:3.0:1761231917.339468:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231917.339477:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231917.339488:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231917.339515:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607030464/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231917.339558:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231917/real 1761231917] req@00000000daa39bb1 x1846780607030272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231942 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231917.339585:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607030272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231942 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231917.339599:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231917.339603:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231917.339608:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607030272:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231917.339618:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030336:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231917.339647:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231917/real 1761231917] req@000000003ad149cc x1846780607030336/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231942 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231917.339659:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607030336/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231942 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231917.339667:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231917.339670:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231917.339673:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030336:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231917.339705:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030400:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231917.339731:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231917/real 1761231917] req@0000000088473b3e x1846780607030400/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231942 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231917.339743:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607030400/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231942 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231917.339753:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231917.339755:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231917.339758:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030400:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231917.339764:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030464:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231917.339785:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231917/real 1761231917] req@000000004e3f612a x1846780607030464/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231942 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231917.339796:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607030464/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231942 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231917.339805:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231917.339808:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231917.339810:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030464:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231917.787133:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231918.811168:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231919.835138:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231920.859030:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231921.883105:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231922.459063:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231922.459074:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231922.459078:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231922.459090:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4794 00000100:00080000:2.0:1761231922.459094:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231922.459101:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231922.459141:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607030528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231922.459173:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231922.459177:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231922.459179:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231922.459184:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4794 00000100:00080000:2.0:1761231922.459186:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231922.459189:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231922.459197:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231922.459211:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607030592/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231922.459220:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231922.459223:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231922.459224:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231922.459228:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4794 00000100:00080000:2.0:1761231922.459229:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231922.459232:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231922.459235:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231922.459246:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607030656/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231922.459256:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231922.459260:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231922.459262:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231922.459266:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4794 00000100:00080000:2.0:1761231922.459268:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231922.459271:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231922.459275:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231922.459286:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607030720/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231922.459293:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607030528:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231922.459397:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030592:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231922.459421:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030656:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231922.459448:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030720:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231922.906997:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231923.931214:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231924.955190:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761231925.531442:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231922/real 1761231925] req@00000000c069b612 x1846780607030720/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231947 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231925.531490:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607030720/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231947 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231925.531510:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231925.531516:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231925.531523:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030720:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231925.531534:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231922/real 1761231925] req@0000000064978143 x1846780607030528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231947 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231925.531556:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607030528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231947 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231925.531570:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231925.531574:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231925.531579:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607030528:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231925.531589:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231922/real 1761231925] req@0000000091106a97 x1846780607030592/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231947 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231925.531609:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607030592/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231947 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231925.531624:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231925.531628:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231925.531633:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030592:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231925.531641:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231922/real 1761231925] req@0000000094135f44 x1846780607030656/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231947 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231925.531659:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607030656/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231947 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231925.531673:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231925.531677:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231925.531681:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030656:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231925.979223:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231927.003193:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761231927.579085:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231927.579094:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761231927.579097:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231927.579105:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4799 00000100:00080000:1.0:1761231927.579108:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231927.579113:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761231927.579142:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607030784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231927.579161:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231927.579164:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761231927.579165:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231927.579168:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4799 00000100:00080000:1.0:1761231927.579169:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231927.579171:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231927.579177:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761231927.579184:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607030848/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231927.579189:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231927.579192:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761231927.579192:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231927.579195:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4799 00000100:00080000:1.0:1761231927.579196:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231927.579197:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231927.579200:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231927.579205:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607030912/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761231927.579209:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761231927.579210:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761231927.579211:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761231927.579213:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4799 00000100:00080000:1.0:1761231927.579214:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761231927.579216:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761231927.579219:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761231927.579225:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607030976/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231927.579253:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607030784:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231927.579342:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231927/real 1761231927] req@00000000b72feabb x1846780607030784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231952 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231927.579359:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607030784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231952 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231927.579369:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231927.579372:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231927.579377:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607030784:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231927.579384:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030848:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231927.579416:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231927/real 1761231927] req@00000000659d78e3 x1846780607030848/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231952 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231927.579427:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607030848/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231952 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231927.579442:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231927.579445:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231927.579448:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030848:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231927.579454:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030912:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231927.579481:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231927/real 1761231927] req@00000000db10af3c x1846780607030912/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231952 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231927.579491:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607030912/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231952 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231927.579498:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231927.579500:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231927.579503:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030912:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231927.579508:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030976:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231927.579531:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231927/real 1761231927] req@00000000d07717c4 x1846780607030976/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231952 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231927.579542:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607030976/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231952 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231927.579552:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231927.579554:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231927.579557:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607030976:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231928.027128:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231929.051104:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231930.075169:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231931.099163:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231932.123170:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231932.699067:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231932.699080:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231932.699086:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231932.699099:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4805 00000100:00080000:3.0:1761231932.699104:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231932.699112:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231932.699170:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607031040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231932.699219:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231932.699223:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231932.699225:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231932.699230:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4805 00000100:00080000:3.0:1761231932.699233:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231932.699236:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231932.699247:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231932.699259:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607031104/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231932.699272:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231932.699277:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231932.699279:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231932.699282:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4805 00000100:00080000:3.0:1761231932.699296:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231932.699300:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231932.699307:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231932.699317:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607031168/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231932.699327:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231932.699332:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231932.699334:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231932.699338:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4805 00000100:00080000:3.0:1761231932.699340:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231932.699344:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231932.699350:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231932.699365:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607031040:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231932.699365:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607031232/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231932.699568:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231932/real 1761231932] req@000000004e3f612a x1846780607031040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231957 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231932.699596:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607031040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231957 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231932.699611:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231932.699616:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231932.699622:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607031040:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231932.699633:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031104:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231932.699663:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231932/real 1761231932] req@0000000088473b3e x1846780607031104/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231957 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231932.699676:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607031104/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231957 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231932.699684:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231932.699687:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231932.699690:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031104:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231932.699698:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031168:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231932.699723:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231932/real 1761231932] req@000000003ad149cc x1846780607031168/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231957 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231932.699735:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607031168/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231957 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231932.699745:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231932.699747:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231932.699750:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031168:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231932.699794:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031232:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231932.699819:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231932/real 1761231932] req@00000000daa39bb1 x1846780607031232/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231957 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231932.699830:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607031232/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231957 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231932.699947:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231932.699952:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231932.699957:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031232:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231933.147008:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231934.171008:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231935.194988:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231936.219035:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231937.243009:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231937.818895:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231937.818904:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231937.818907:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231937.818917:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4810 00000100:00080000:3.0:1761231937.818920:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231937.818926:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231937.818956:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607031296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231937.818975:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231937.818978:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231937.818979:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231937.818982:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4810 00000100:00080000:3.0:1761231937.818983:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231937.818985:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231937.818992:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231937.818999:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607031360/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231937.819006:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761231937.819009:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607031296:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761231937.819010:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231937.819012:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231937.819017:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4810 00000100:00080000:3.0:1761231937.819019:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231937.819020:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231937.819024:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231937.819032:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607031424/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231937.819037:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231937.819039:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231937.819040:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231937.819043:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4810 00000100:00080000:3.0:1761231937.819045:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231937.819047:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231937.819049:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231937.819056:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607031488/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231937.819090:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231937/real 1761231937] req@0000000086ccb527 x1846780607031296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231962 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231937.819104:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607031296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231962 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231937.819111:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231937.819113:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231937.819116:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607031296:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231937.819133:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031360:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231937.819148:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231937/real 1761231937] req@000000005a9bdefd x1846780607031360/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231962 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231937.819154:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607031360/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231962 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231937.819159:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231937.819160:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231937.819161:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031360:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231937.819165:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031424:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231937.819176:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231937/real 1761231937] req@0000000018a8f5da x1846780607031424/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231962 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231937.819182:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607031424/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231962 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231937.819187:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231937.819188:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231937.819190:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031424:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231937.819193:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031488:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231937.819204:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231937/real 1761231937] req@00000000bd684ccc x1846780607031488/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231962 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231937.819209:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607031488/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231962 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231937.819214:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231937.819215:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231937.819217:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031488:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231938.267047:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231939.291016:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231940.315002:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231941.339052:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231942.363022:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231942.938910:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231942.938918:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231942.938922:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231942.938931:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4815 00000100:00080000:3.0:1761231942.938933:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231942.938938:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231942.938969:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607031552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231942.938988:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231942.938990:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231942.938991:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231942.938994:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4815 00000100:00080000:3.0:1761231942.938996:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231942.938998:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231942.939004:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231942.939010:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607031616/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231942.939016:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231942.939020:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231942.939021:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231942.939023:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4815 00000100:00080000:3.0:1761231942.939024:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231942.939027:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231942.939029:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231942.939038:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607031552:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231942.939039:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607031680/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231942.939051:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231942.939052:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231942.939053:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231942.939055:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4815 00000100:00080000:3.0:1761231942.939056:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231942.939058:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231942.939061:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231942.939069:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607031744/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231942.939124:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231942/real 1761231942] req@00000000c7e4e432 x1846780607031552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231967 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231942.939142:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607031552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231967 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231942.939152:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231942.939155:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231942.939158:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607031552:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231942.939165:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031616:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231942.939187:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231942/real 1761231942] req@00000000f87d0618 x1846780607031616/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231967 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231942.939199:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607031616/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231967 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231942.939206:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231942.939208:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231942.939211:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031616:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231942.939235:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031680:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231942.939255:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231942/real 1761231942] req@000000007361665d x1846780607031680/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231967 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231942.939267:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607031680/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231967 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231942.939275:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231942.939277:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231942.939280:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031680:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231942.939284:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031744:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231942.939301:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231942/real 1761231942] req@0000000031e043f4 x1846780607031744/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231967 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231942.939311:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607031744/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231967 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231942.939319:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231942.939321:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231942.939325:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031744:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231943.387053:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231944.411074:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231945.435043:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231946.459049:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231947.483106:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231948.058996:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231948.059005:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231948.059008:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231948.059017:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4820 00000100:00080000:2.0:1761231948.059019:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231948.059025:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231948.059056:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607031808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231948.059079:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231948.059081:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231948.059082:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231948.059086:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4820 00000100:00080000:2.0:1761231948.059088:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231948.059090:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231948.059097:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231948.059105:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607031872/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231948.059110:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231948.059112:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231948.059113:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231948.059115:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4820 00000100:00080000:2.0:1761231948.059116:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231948.059118:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231948.059120:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231948.059126:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607031936/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231948.059130:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231948.059132:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231948.059133:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231948.059135:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4820 00000100:00080000:2.0:1761231948.059136:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231948.059139:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231948.059142:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231948.059147:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607032000/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231948.059160:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607031808:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231948.059235:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231948/real 1761231948] req@0000000017386078 x1846780607031808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231973 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231948.059246:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607031808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231973 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231948.059254:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231948.059256:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231948.059258:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607031808:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231948.059263:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031872:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231948.059277:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231948/real 1761231948] req@00000000f38f27df x1846780607031872/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231973 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231948.059284:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607031872/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231973 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231948.059288:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231948.059289:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231948.059291:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031872:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231948.059294:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031936:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231948.059307:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231948/real 1761231948] req@0000000094135f44 x1846780607031936/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231973 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231948.059313:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607031936/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231973 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231948.059318:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231948.059319:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231948.059320:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607031936:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231948.059339:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032000:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231948.059351:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231948/real 1761231948] req@0000000091106a97 x1846780607032000/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231973 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231948.059356:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607032000/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231973 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231948.059361:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231948.059362:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231948.059363:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032000:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231948.507045:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231949.531053:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231950.554988:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231951.579021:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231952.603051:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231953.178961:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231953.178972:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231953.178976:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231953.178986:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4825 00000100:00080000:0.0:1761231953.178990:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231953.178997:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231953.179033:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607032064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231953.179059:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231953.179062:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231953.179063:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231953.179066:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4825 00000100:00080000:0.0:1761231953.179068:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231953.179070:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231953.179076:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231953.179084:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607032128/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231953.179089:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231953.179091:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231953.179092:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231953.179094:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4825 00000100:00080000:0.0:1761231953.179095:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231953.179097:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231953.179099:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231953.179104:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607032192/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231953.179108:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231953.179111:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231953.179112:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231953.179114:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4825 00000100:00080000:0.0:1761231953.179116:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231953.179118:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231953.179120:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231953.179128:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607032256/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231953.179145:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607032064:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231953.179218:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231953/real 1761231953] req@000000005a096598 x1846780607032064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231978 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231953.179229:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607032064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231978 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231953.179237:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231953.179239:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231953.179243:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607032064:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231953.179248:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032128:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231953.179262:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231953/real 1761231953] req@00000000275aeee5 x1846780607032128/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231978 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231953.179268:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607032128/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231978 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231953.179273:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231953.179274:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231953.179275:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032128:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231953.179278:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032192:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231953.179290:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231953/real 1761231953] req@000000002ef1f54f x1846780607032192/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231978 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231953.179297:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607032192/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231978 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231953.179302:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231953.179303:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231953.179304:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032192:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231953.179307:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032256:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231953.179318:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231953/real 1761231953] req@0000000023ab3a3f x1846780607032256/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231978 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231953.179328:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607032256/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231978 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231953.179333:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231953.179334:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231953.179335:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032256:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231953.627052:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231954.651009:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231955.675180:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231956.699033:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231957.723129:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231958.299029:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231958.299050:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231958.299056:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231958.299072:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4830 00000100:00080000:0.0:1761231958.299077:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231958.299087:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231958.299142:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607032320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231958.299186:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231958.299193:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231958.299196:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231958.299203:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4830 00000100:00080000:0.0:1761231958.299208:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231958.299217:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231958.299233:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231958.299236:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607032320:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231958.299263:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607032384/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231958.299279:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231958.299284:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231958.299286:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231958.299291:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4830 00000100:00080000:0.0:1761231958.299294:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231958.299298:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231958.299304:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231958.299320:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607032448/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231958.299332:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231958.299337:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231958.299339:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231958.299344:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4830 00000100:00080000:0.0:1761231958.299347:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231958.299351:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231958.299356:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231958.299357:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231958/real 1761231958] req@00000000271a9d8e x1846780607032320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231983 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231958.299373:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607032512/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231958.299387:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607032320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231983 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231958.299402:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231958.299407:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231958.299412:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607032320:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231958.299446:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032384:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231958.299483:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231958/real 1761231958] req@000000000243251f x1846780607032384/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231983 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231958.299495:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607032384/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231983 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231958.299504:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231958.299507:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231958.299510:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032384:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231958.299517:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032448:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231958.299540:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231958/real 1761231958] req@0000000099514377 x1846780607032448/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231983 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231958.299553:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607032448/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231983 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231958.299561:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231958.299564:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231958.299566:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032448:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231958.299572:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032512:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231958.299592:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231958/real 1761231958] req@00000000fcce404c x1846780607032512/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231983 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231958.299640:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607032512/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231983 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231958.299659:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231958.299661:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231958.299664:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032512:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231958.747168:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231959.771094:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231960.795181:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231961.819117:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231962.843156:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231963.419063:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231963.419077:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231963.419083:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231963.419096:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4835 00000100:00080000:3.0:1761231963.419100:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231963.419110:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231963.419157:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607032576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231963.419198:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231963.419202:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231963.419204:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231963.419209:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4835 00000100:00080000:3.0:1761231963.419212:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231963.419216:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231963.419227:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231963.419242:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607032640/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231963.419255:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231963.419261:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1761231963.419263:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607032576:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761231963.419264:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231963.419277:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4835 00000100:00080000:3.0:1761231963.419280:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231963.419284:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231963.419289:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231963.419304:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607032704/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231963.419315:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231963.419319:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231963.419321:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231963.419326:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4835 00000100:00080000:3.0:1761231963.419328:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231963.419333:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231963.419338:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231963.419350:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607032768/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231963.419391:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231963/real 1761231963] req@000000001f8e688e x1846780607032576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231988 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231963.419415:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607032576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231988 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231963.419428:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231963.419432:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231963.419437:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607032576:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231963.419466:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032640:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231963.419495:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231963/real 1761231963] req@0000000090b40c66 x1846780607032640/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231988 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231963.419508:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607032640/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231988 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231963.419516:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231963.419518:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231963.419521:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032640:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231963.419527:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032704:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231963.419588:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231963/real 1761231963] req@0000000083f16a6b x1846780607032704/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231988 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231963.419603:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607032704/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231988 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231963.419612:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231963.419616:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231963.419619:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032704:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231963.419625:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032768:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231963.419652:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231963/real 1761231963] req@00000000a58ca11e x1846780607032768/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231988 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231963.419665:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607032768/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231988 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231963.419673:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231963.419675:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231963.419678:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032768:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231963.867193:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231964.891135:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231965.915128:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231966.939164:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231967.962990:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231968.539054:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231968.539065:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231968.539070:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231968.539082:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4840 00000100:00080000:0.0:1761231968.539086:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231968.539094:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231968.539133:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607032832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231968.539172:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231968.539176:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231968.539179:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231968.539184:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4840 00000100:00080000:0.0:1761231968.539186:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231968.539190:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231968.539201:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231968.539217:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607032896/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231968.539231:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761231968.539237:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607032832:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:0.0:1761231968.539239:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231968.539244:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231968.539254:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4840 00000100:00080000:0.0:1761231968.539257:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231968.539260:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231968.539266:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231968.539283:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607032960/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231968.539294:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231968.539299:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231968.539300:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231968.539304:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4840 00000100:00080000:0.0:1761231968.539307:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231968.539310:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231968.539314:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231968.539327:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607033024/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231968.539420:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231968/real 1761231968] req@000000007f60e210 x1846780607032832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231993 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231968.539442:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607032832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231993 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231968.539454:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231968.539458:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231968.539463:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607032832:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231968.539512:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032896:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231968.539549:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231968/real 1761231968] req@0000000001b2f813 x1846780607032896/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231993 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231968.539563:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607032896/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231993 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231968.539572:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231968.539575:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231968.539578:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032896:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231968.539586:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032960:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231968.539609:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231968/real 1761231968] req@00000000b8f2be78 x1846780607032960/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231993 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231968.539621:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607032960/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231993 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231968.539630:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231968.539632:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231968.539635:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607032960:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231968.539640:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033024:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231968.539662:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231968/real 1761231968] req@00000000fcce404c x1846780607033024/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231993 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231968.539674:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607033024/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231993 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231968.539682:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231968.539684:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231968.539687:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033024:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231968.987123:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231970.011102:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231971.035150:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231972.059185:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231973.083126:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231973.659099:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231973.659112:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231973.659117:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231973.659131:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4846 00000100:00080000:3.0:1761231973.659136:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231973.659145:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231973.659190:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607033088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231973.659233:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231973.659238:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231973.659240:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231973.659245:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4846 00000100:00080000:3.0:1761231973.659247:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231973.659251:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231973.659262:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231973.659282:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607033152/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231973.659295:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231973.659300:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231973.659301:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231973.659305:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4846 00000100:00080000:3.0:1761231973.659307:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231973.659310:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231973.659317:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231973.659331:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607033088:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231973.659332:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607033216/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231973.659369:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231973.659373:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231973.659374:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231973.659382:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4846 00000100:00080000:3.0:1761231973.659383:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231973.659388:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231973.659393:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231973.659405:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607033280/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231973.659554:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231973/real 1761231973] req@000000002ce934d3 x1846780607033088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231998 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231973.659581:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607033088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761231998 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231973.659597:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231973.659602:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231973.659607:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607033088:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231973.659618:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033152:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231973.659649:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231973/real 1761231973] req@000000006572b591 x1846780607033152/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231998 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231973.659662:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607033152/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761231998 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231973.659671:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231973.659673:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231973.659676:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033152:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761231973.659717:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033216:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231973.659743:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231973/real 1761231973] req@00000000fc01c41c x1846780607033216/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231998 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231973.659756:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607033216/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231998 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231973.659766:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231973.659768:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231973.659771:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033216:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231973.659778:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033280:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761231973.659799:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231973/real 1761231973] req@00000000408f088c x1846780607033280/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231998 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761231973.659811:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607033280/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761231998 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761231973.659820:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231973.659823:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231973.659826:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033280:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231974.107057:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231975.131186:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231976.154962:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231977.179006:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231978.203115:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761231978.778959:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231978.778976:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761231978.778983:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231978.778998:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4851 00000100:00080000:3.0:1761231978.779005:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231978.779016:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761231978.779070:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607033344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231978.779111:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231978.779117:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761231978.779120:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231978.779127:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4851 00000100:00080000:3.0:1761231978.779130:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231978.779137:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231978.779151:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761231978.779172:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607033408/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231978.779189:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231978.779195:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761231978.779198:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231978.779203:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4851 00000100:00100000:0.0:1761231978.779204:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607033344:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761231978.779207:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231978.779213:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231978.779219:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231978.779235:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607033472/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761231978.779250:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761231978.779255:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761231978.779257:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761231978.779262:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4851 00000100:00080000:3.0:1761231978.779265:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761231978.779270:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761231978.779276:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231978.779289:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607033536/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761231978.779329:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231978/real 1761231978] req@00000000408f088c x1846780607033344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232003 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231978.779358:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607033344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232003 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231978.779411:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231978.779418:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231978.779425:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607033344:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761231978.779438:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033408:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231978.779493:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231978/real 1761231978] req@00000000fc01c41c x1846780607033408/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232003 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231978.779514:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607033408/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232003 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231978.779528:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231978.779532:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231978.779537:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033408:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761231978.779576:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033472:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231978.779612:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231978/real 1761231978] req@000000006572b591 x1846780607033472/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232003 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231978.779625:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607033472/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232003 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231978.779635:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231978.779637:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231978.779640:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033472:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231978.779646:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033536:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:0.0:1761231978.779668:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231978/real 1761231978] req@000000002ce934d3 x1846780607033536/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232003 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761231978.779679:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607033536/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232003 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761231978.779688:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761231978.779690:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761231978.779693:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033536:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231979.227066:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231980.251180:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231981.275137:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231982.299123:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231983.323175:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761231983.899060:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231983.899074:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761231983.899078:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231983.899092:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4856 00000100:00080000:0.0:1761231983.899096:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231983.899105:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761231983.899150:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607033600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231983.899190:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231983.899195:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761231983.899197:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231983.899202:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4856 00000100:00080000:0.0:1761231983.899204:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231983.899209:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231983.899219:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761231983.899234:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607033664/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231983.899245:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231983.899249:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761231983.899252:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231983.899256:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4856 00000100:00080000:0.0:1761231983.899258:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231983.899261:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231983.899267:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761231983.899269:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607033600:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761231983.899282:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607033728/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761231983.899295:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761231983.899299:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761231983.899302:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761231983.899305:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4856 00000100:00080000:0.0:1761231983.899308:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761231983.899311:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761231983.899378:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761231983.899396:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607033792/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231983.899419:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231983/real 1761231983] req@0000000099514377 x1846780607033600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232008 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231983.899446:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607033600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232008 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231983.899460:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231983.899465:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231983.899470:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607033600:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231983.899479:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033664:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231983.899508:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231983/real 1761231983] req@000000000243251f x1846780607033664/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232008 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231983.899521:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607033664/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232008 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231983.899530:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231983.899533:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231983.899536:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033664:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231983.899564:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033728:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231983.899587:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231983/real 1761231983] req@00000000271a9d8e x1846780607033728/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232008 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231983.899599:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607033728/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232008 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231983.899608:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231983.899611:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231983.899614:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033728:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231983.899620:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033792:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231983.899640:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231983/real 1761231983] req@0000000023ab3a3f x1846780607033792/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232008 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231983.899660:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607033792/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232008 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231983.899669:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231983.899671:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231983.899674:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033792:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231984.347041:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231985.371137:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231986.395130:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231987.419146:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231988.443162:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231989.018990:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231989.019004:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231989.019009:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231989.019023:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4861 00000100:00080000:2.0:1761231989.019028:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231989.019036:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231989.019083:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607033856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231989.019119:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231989.019126:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231989.019128:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231989.019135:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4861 00000100:00080000:2.0:1761231989.019138:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231989.019144:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231989.019155:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231989.019174:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607033920/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231989.019189:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231989.019197:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231989.019199:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231989.019205:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4861 00000100:00080000:2.0:1761231989.019209:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1761231989.019212:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607033856:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:2.0:1761231989.019219:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231989.019229:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231989.019252:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607033984/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231989.019289:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231989.019295:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231989.019297:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231989.019303:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4861 00000100:00080000:2.0:1761231989.019306:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231989.019312:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231989.019320:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231989.019339:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607034048/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231989.019343:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033920:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231989.019382:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033984:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231989.019402:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034048:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231989.467124:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231990.491159:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231991.515062:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761231992.091517:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231989/real 1761231992] req@0000000091106a97 x1846780607033984/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232014 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231992.091567:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607033984/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232014 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231992.091589:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231992.091596:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231992.091604:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033984:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231992.091616:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231989/real 1761231992] req@0000000094135f44 x1846780607034048/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232014 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231992.091653:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607034048/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232014 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231992.091669:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231992.091673:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231992.091679:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034048:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231992.091690:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231989/real 1761231992] req@0000000064978143 x1846780607033856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232014 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231992.091714:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607033856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232014 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231992.091728:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231992.091732:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231992.091737:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607033856:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231992.091747:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231989/real 1761231992] req@00000000c069b612 x1846780607033920/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232014 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231992.091765:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607033920/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232014 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231992.091777:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231992.091782:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231992.091787:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607033920:192.168.201.104@tcp:38:kworker/u8:0.0 00010000:00010000:0.0:1761231992.539193:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231993.563042:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231994.138945:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231994.138956:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231994.138960:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231994.138970:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4866 00000100:00080000:2.0:1761231994.138975:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231994.138981:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231994.139016:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607034112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231994.139041:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231994.139044:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231994.139045:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231994.139049:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4866 00000100:00080000:2.0:1761231994.139050:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231994.139052:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231994.139060:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231994.139068:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607034176/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231994.139075:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231994.139077:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231994.139078:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231994.139081:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4866 00000100:00080000:2.0:1761231994.139082:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231994.139084:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231994.139087:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231994.139094:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607034240/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231994.139100:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231994.139102:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231994.139103:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231994.139106:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4866 00000100:00080000:2.0:1761231994.139109:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231994.139112:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231994.139115:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231994.139123:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607034304/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231994.139135:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607034112:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231994.139264:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231994/real 1761231994] req@00000000f38f27df x1846780607034112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232019 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231994.139285:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607034112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232019 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231994.139298:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231994.139302:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231994.139306:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607034112:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761231994.139315:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034176:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231994.139359:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231994/real 1761231994] req@0000000017386078 x1846780607034176/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232019 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231994.139374:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607034176/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232019 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231994.139383:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231994.139386:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231994.139390:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034176:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761231994.139397:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034240:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231994.139429:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231994/real 1761231994] req@00000000c069b612 x1846780607034240/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232019 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231994.139442:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607034240/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232019 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231994.139451:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231994.139454:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231994.139457:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034240:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231994.139488:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034304:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761231994.139520:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231994/real 1761231994] req@0000000064978143 x1846780607034304/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232019 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761231994.139534:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607034304/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232019 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761231994.139544:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231994.139546:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231994.139550:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034304:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761231994.587138:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231995.610994:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761231996.635108:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231997.659135:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761231998.683132:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761231999.259014:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231999.259031:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761231999.259038:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231999.259055:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4871 00000100:00080000:2.0:1761231999.259061:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231999.259072:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761231999.259134:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607034368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231999.259222:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231999.259227:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761231999.259229:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231999.259235:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4871 00000100:00080000:2.0:1761231999.259238:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231999.259242:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231999.259256:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761231999.259273:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607034432/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231999.259284:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231999.259291:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761231999.259294:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231999.259299:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4871 00000100:00080000:2.0:1761231999.259302:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231999.259306:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231999.259311:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231999.259321:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607034496/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761231999.259331:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761231999.259335:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761231999.259337:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761231999.259341:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4871 00000100:00080000:2.0:1761231999.259344:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761231999.259349:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761231999.259354:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761231999.259364:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607034560/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761231999.259408:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607034368:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231999.259554:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231999/real 1761231999] req@0000000094135f44 x1846780607034368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232024 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231999.259577:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607034368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232024 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231999.259590:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231999.259595:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231999.259600:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607034368:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761231999.259614:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034432:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231999.259665:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231999/real 1761231999] req@0000000091106a97 x1846780607034432/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232024 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231999.259689:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607034432/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232024 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231999.259705:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231999.259711:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231999.259717:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034432:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761231999.259730:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034496:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231999.259778:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231999/real 1761231999] req@0000000064978143 x1846780607034496/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232024 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231999.259793:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607034496/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232024 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231999.259803:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231999.259806:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231999.259810:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034496:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231999.259817:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034560:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761231999.259910:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761231999/real 1761231999] req@00000000c069b612 x1846780607034560/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232024 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761231999.259925:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607034560/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232024 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761231999.259934:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761231999.259937:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761231999.259940:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034560:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761231999.707224:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232000.731190:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232001.755197:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232002.778976:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232003.803168:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232004.378993:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232004.379010:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232004.379017:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232004.379035:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4876 00000100:00080000:2.0:1761232004.379040:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232004.379052:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232004.379143:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607034624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232004.379191:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232004.379197:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232004.379199:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232004.379204:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4876 00000100:00080000:2.0:1761232004.379207:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232004.379211:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232004.379224:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232004.379242:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607034688/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232004.379253:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232004.379258:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232004.379261:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232004.379267:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4876 00000100:00080000:2.0:1761232004.379270:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232004.379274:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232004.379281:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232004.379292:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607034752/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232004.379306:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232004.379312:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232004.379316:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761232004.379320:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607034624:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761232004.379325:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4876 00000100:00080000:2.0:1761232004.379334:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232004.379344:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232004.379350:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232004.379368:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607034816/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232004.379457:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232004/real 1761232004] req@0000000017386078 x1846780607034624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232029 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232004.379489:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607034624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232029 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232004.379501:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232004.379505:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232004.379510:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607034624:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232004.379519:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034688:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232004.379548:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232004/real 1761232004] req@00000000f38f27df x1846780607034688/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232029 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232004.379561:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607034688/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232029 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232004.379569:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232004.379572:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232004.379575:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034688:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232004.379607:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034752:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232004.379650:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232004/real 1761232004] req@00000000c069b612 x1846780607034752/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232029 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232004.379664:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607034752/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232029 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232004.379673:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232004.379676:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232004.379679:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034752:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232004.379687:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034816:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232004.379720:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232004/real 1761232004] req@0000000064978143 x1846780607034816/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232029 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232004.379733:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607034816/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232029 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232004.379742:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232004.379744:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232004.379747:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034816:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232004.827209:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232005.851233:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232006.875148:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232007.899144:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232008.923165:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232009.498999:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232009.499019:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232009.499073:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232009.499092:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4881 00000100:00080000:2.0:1761232009.499099:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232009.499108:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232009.499157:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607034880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232009.499200:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232009.499205:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232009.499207:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232009.499212:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4881 00000100:00080000:2.0:1761232009.499214:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232009.499219:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232009.499230:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232009.499246:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607034944/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232009.499257:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232009.499265:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232009.499268:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232009.499273:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4881 00000100:00080000:2.0:1761232009.499275:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232009.499279:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232009.499285:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232009.499301:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607034880:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761232009.499302:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607035008/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232009.499324:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232009.499327:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232009.499328:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232009.499333:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4881 00000100:00080000:2.0:1761232009.499335:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232009.499339:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232009.499344:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232009.499358:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607035072/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232009.499425:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232009/real 1761232009] req@0000000091106a97 x1846780607034880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232034 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232009.499447:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607034880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232034 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232009.499460:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232009.499463:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232009.499468:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607034880:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232009.499477:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034944:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232009.499505:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232009/real 1761232009] req@0000000094135f44 x1846780607034944/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232034 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232009.499517:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607034944/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232034 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232009.499534:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232009.499537:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232009.499540:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607034944:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232009.499568:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035008:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232009.499591:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232009/real 1761232009] req@0000000064978143 x1846780607035008/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232034 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232009.499603:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607035008/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232034 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232009.499613:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232009.499615:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232009.499618:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035008:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232009.499624:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035072:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232009.499643:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232009/real 1761232009] req@00000000c069b612 x1846780607035072/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232034 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232009.499654:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607035072/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232034 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232009.499662:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232009.499664:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232009.499668:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035072:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232009.946990:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232010.971044:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232011.995211:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232013.019184:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232014.043277:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232014.618924:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232014.618933:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232014.618936:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232014.618945:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4887 00000100:00080000:2.0:1761232014.618948:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232014.618953:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232014.619001:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607035136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232014.619039:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232014.619041:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232014.619042:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232014.619046:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4887 00000100:00080000:2.0:1761232014.619047:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232014.619049:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232014.619056:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232014.619066:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607035200/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232014.619072:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232014.619074:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232014.619076:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232014.619078:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4887 00000100:00080000:2.0:1761232014.619079:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232014.619081:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232014.619084:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232014.619088:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607035136:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761232014.619093:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607035264/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232014.619101:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232014.619103:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232014.619104:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232014.619106:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4887 00000100:00080000:2.0:1761232014.619108:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232014.619111:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232014.619114:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232014.619121:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607035328/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232014.619180:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232014/real 1761232014] req@00000000f38f27df x1846780607035136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232039 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232014.619193:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607035136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232039 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232014.619201:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232014.619203:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232014.619206:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607035136:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232014.619211:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035200:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232014.619229:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232014/real 1761232014] req@0000000017386078 x1846780607035200/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232039 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232014.619236:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607035200/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232039 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232014.619240:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232014.619241:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232014.619243:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035200:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232014.619261:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035264:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232014.619273:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232014/real 1761232014] req@00000000c069b612 x1846780607035264/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232039 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232014.619280:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607035264/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232039 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232014.619285:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232014.619287:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232014.619288:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035264:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232014.619291:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035328:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232014.619302:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232014/real 1761232014] req@0000000064978143 x1846780607035328/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232039 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232014.619308:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607035328/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232039 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232014.619312:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232014.619313:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232014.619315:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035328:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232015.067032:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232016.091129:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232017.115147:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232018.139084:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232019.163150:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232019.739035:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232019.739049:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232019.739054:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232019.739067:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4892 00000100:00080000:0.0:1761232019.739071:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232019.739081:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232019.739127:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607035392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232019.739167:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232019.739172:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232019.739175:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232019.739179:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4892 00000100:00080000:0.0:1761232019.739182:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232019.739186:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232019.739196:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232019.739209:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607035456/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232019.739220:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232019.739224:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232019.739225:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232019.739229:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4892 00000100:00080000:0.0:1761232019.739231:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232019.739234:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232019.739238:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232019.739247:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607035520/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232019.739256:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232019.739259:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232019.739261:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232019.739266:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4892 00000100:00080000:0.0:1761232019.739270:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232019.739274:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232019.739278:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232019.739289:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607035584/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232019.739308:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607035392:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232019.739419:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232019/real 1761232019] req@000000002ef1f54f x1846780607035392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232044 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232019.739441:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607035392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232044 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232019.739453:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232019.739457:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232019.739462:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607035392:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232019.739470:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035456:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232019.739496:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232019/real 1761232019] req@00000000275aeee5 x1846780607035456/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232044 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232019.739517:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607035456/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232044 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232019.739526:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232019.739528:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232019.739531:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035456:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232019.739537:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035520:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232019.739559:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232019/real 1761232019] req@000000005a096598 x1846780607035520/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232044 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232019.739570:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607035520/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232044 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232019.739579:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232019.739581:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232019.739584:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035520:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232019.739616:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035584:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232019.739638:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232019/real 1761232019] req@0000000023ab3a3f x1846780607035584/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232044 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232019.739649:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607035584/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232044 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232019.739658:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232019.739660:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232019.739664:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035584:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232020.187147:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232021.211204:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232022.235116:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232023.259134:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232024.283172:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232024.858978:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232024.859003:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232024.859011:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232024.859033:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4897 00000100:00080000:0.0:1761232024.859041:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232024.859054:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232024.859119:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607035648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232024.859160:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232024.859165:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232024.859167:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232024.859173:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4897 00000100:00080000:0.0:1761232024.859176:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232024.859180:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232024.859194:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232024.859214:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607035712/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232024.859228:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232024.859233:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232024.859236:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761232024.859240:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607035648:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:0.0:1761232024.859245:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4897 00000100:00080000:0.0:1761232024.859252:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232024.859261:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232024.859268:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232024.859283:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607035776/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232024.859293:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232024.859298:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232024.859300:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232024.859304:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4897 00000100:00080000:0.0:1761232024.859305:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232024.859309:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232024.859313:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232024.859327:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607035840/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232024.859434:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232024/real 1761232024] req@00000000271a9d8e x1846780607035648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232049 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232024.859469:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607035648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232049 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232024.859490:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232024.859496:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232024.859504:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607035648:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232024.859553:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035712:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232024.859613:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232024/real 1761232024] req@000000000243251f x1846780607035712/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232049 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232024.859645:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607035712/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232049 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232024.859662:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232024.859666:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232024.859672:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035712:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232024.859683:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035776:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232024.859729:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232024/real 1761232024] req@0000000099514377 x1846780607035776/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232049 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232024.859750:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607035776/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232049 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232024.859769:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232024.859774:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232024.859781:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035776:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232024.859792:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035840:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232024.859914:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232024/real 1761232024] req@00000000fcce404c x1846780607035840/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232049 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232024.859937:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607035840/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232049 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232024.859953:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232024.859957:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232024.859963:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035840:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232025.307127:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232026.331190:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232027.355160:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232028.379194:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232029.403079:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232029.978949:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232029.978964:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232029.978969:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232029.978981:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4902 00000100:00080000:0.0:1761232029.978985:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232029.978994:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232029.979037:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607035904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232029.979071:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232029.979075:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232029.979077:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232029.979082:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4902 00000100:00080000:0.0:1761232029.979084:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232029.979087:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232029.979097:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232029.979109:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607035968/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232029.979119:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232029.979122:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232029.979124:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232029.979128:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4902 00000100:00080000:0.0:1761232029.979131:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232029.979134:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232029.979138:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232029.979151:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607036032/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232029.979162:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232029.979166:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232029.979168:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232029.979172:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4902 00000100:00080000:0.0:1761232029.979174:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232029.979178:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232029.979182:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232029.979193:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607035904:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761232029.979198:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607036096/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232029.979308:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232029/real 1761232029] req@00000000b8f2be78 x1846780607035904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232054 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232029.979329:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607035904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232054 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232029.979342:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232029.979346:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232029.979350:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607035904:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232029.979358:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035968:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232029.979389:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232029/real 1761232029] req@0000000001b2f813 x1846780607035968/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232054 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232029.979401:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607035968/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232054 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232029.979409:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232029.979412:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232029.979415:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607035968:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232029.979421:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036032:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232029.979441:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232029/real 1761232029] req@000000007f60e210 x1846780607036032/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232054 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232029.979453:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607036032/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232054 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232029.979462:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232029.979464:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232029.979467:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036032:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232029.979498:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036096:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232029.979520:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232029/real 1761232029] req@00000000fcce404c x1846780607036096/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232054 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232029.979531:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607036096/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232054 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232029.979539:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232029.979542:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232029.979545:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036096:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232030.427105:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232031.451066:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232032.475087:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232033.499089:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232034.523116:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232035.099088:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232035.099102:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232035.099107:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232035.099119:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4907 00000100:00080000:1.0:1761232035.099124:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232035.099132:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232035.099181:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607036160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232035.099216:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232035.099221:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232035.099222:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232035.099227:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4907 00000100:00080000:1.0:1761232035.099230:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232035.099233:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232035.099243:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232035.099256:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607036224/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232035.099266:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232035.099270:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232035.099271:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232035.099275:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4907 00000100:00080000:1.0:1761232035.099277:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232035.099279:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232035.099284:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232035.099293:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607036288/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232035.099303:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232035.099307:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232035.099309:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232035.099312:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4907 00000100:00080000:1.0:1761232035.099315:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232035.099319:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232035.099324:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232035.099334:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607036352/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232035.099342:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607036160:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232035.099455:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232035/real 1761232035] req@00000000d07717c4 x1846780607036160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232060 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232035.099477:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607036160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232060 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232035.099489:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232035.099493:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232035.099498:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607036160:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232035.099506:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036224:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232035.099531:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232035/real 1761232035] req@00000000db10af3c x1846780607036224/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232060 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232035.099542:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607036224/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232060 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232035.099551:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232035.099554:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232035.099557:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036224:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232035.099563:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036288:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232035.099583:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232035/real 1761232035] req@00000000659d78e3 x1846780607036288/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232060 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232035.099603:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607036288/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232060 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232035.099612:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232035.099615:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232035.099617:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036288:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232035.099648:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036352:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232035.099671:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232035/real 1761232035] req@00000000b72feabb x1846780607036352/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232060 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232035.099682:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607036352/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232060 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232035.099691:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232035.099693:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232035.099696:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036352:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232035.547117:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232036.571045:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232037.595001:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232038.619098:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232039.643076:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232040.218993:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232040.219006:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232040.219012:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232040.219025:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4912 00000100:00080000:1.0:1761232040.219030:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232040.219039:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232040.219087:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607036416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232040.219121:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232040.219126:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232040.219128:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232040.219133:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4912 00000100:00080000:1.0:1761232040.219135:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232040.219139:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232040.219149:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232040.219162:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607036480/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232040.219173:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232040.219176:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232040.219178:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232040.219184:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4912 00000100:00080000:1.0:1761232040.219186:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232040.219190:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232040.219194:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232040.219205:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607036544/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232040.219215:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232040.219220:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232040.219222:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232040.219227:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4912 00000100:00080000:1.0:1761232040.219229:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232040.219232:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232040.219237:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232040.219247:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607036608/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232040.219255:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607036416:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232040.219370:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232040/real 1761232040] req@00000000b72feabb x1846780607036416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232065 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232040.219391:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607036416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232065 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232040.219404:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232040.219408:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232040.219412:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607036416:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232040.219422:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036480:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232040.219448:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232040/real 1761232040] req@00000000659d78e3 x1846780607036480/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232065 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232040.219460:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607036480/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232065 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232040.219468:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232040.219471:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232040.219474:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036480:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232040.219480:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036544:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232040.219501:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232040/real 1761232040] req@00000000db10af3c x1846780607036544/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232065 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232040.219513:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607036544/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232065 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232040.219522:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232040.219524:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232040.219527:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036544:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232040.219559:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036608:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232040.219581:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232040/real 1761232040] req@00000000d07717c4 x1846780607036608/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232065 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232040.219592:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607036608/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232065 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232040.219626:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232040.219630:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232040.219636:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036608:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232040.667048:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232041.691002:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232042.715046:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232043.739149:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232044.763226:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232045.339088:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232045.339101:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232045.339106:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232045.339118:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4917 00000100:00080000:2.0:1761232045.339123:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232045.339131:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232045.339178:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607036672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232045.339214:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232045.339218:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232045.339220:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232045.339225:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4917 00000100:00080000:2.0:1761232045.339228:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232045.339231:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232045.339241:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232045.339259:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607036736/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232045.339271:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232045.339275:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232045.339277:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232045.339281:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4917 00000100:00080000:2.0:1761232045.339283:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232045.339291:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1761232045.339291:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607036672:192.168.201.104@tcp:250:kworker/u8:2.0 00000008:00100000:2.0:1761232045.339303:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232045.339319:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607036800/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232045.339330:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232045.339334:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232045.339336:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232045.339340:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4917 00000100:00080000:2.0:1761232045.339343:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232045.339347:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232045.339352:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232045.339364:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607036864/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232045.339416:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232045/real 1761232045] req@0000000094135f44 x1846780607036672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232070 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232045.339438:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607036672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232070 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232045.339452:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232045.339456:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232045.339463:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607036672:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232045.339476:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036736:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232045.339524:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232045/real 1761232045] req@0000000091106a97 x1846780607036736/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232070 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232045.339543:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607036736/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232070 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232045.339555:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232045.339560:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232045.339565:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036736:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232045.339604:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036800:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232045.339713:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232045/real 1761232045] req@0000000064978143 x1846780607036800/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232070 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232045.339732:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607036800/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232070 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232045.339746:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232045.339750:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232045.339754:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036800:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232045.339763:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036864:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232045.339805:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232045/real 1761232045] req@00000000c069b612 x1846780607036864/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232070 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232045.339824:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607036864/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232070 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232045.339888:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232045.339892:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232045.339899:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036864:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232045.787117:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232046.811148:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232047.835080:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232048.859216:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232049.883017:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232050.459052:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232050.459067:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232050.459073:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232050.459085:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4922 00000100:00080000:0.0:1761232050.459090:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232050.459098:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232050.459146:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607036928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232050.459181:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232050.459185:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232050.459187:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232050.459192:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4922 00000100:00080000:0.0:1761232050.459195:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232050.459198:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232050.459215:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232050.459228:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607036992/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232050.459239:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232050.459243:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232050.459246:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232050.459251:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4922 00000100:00080000:0.0:1761232050.459253:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232050.459257:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232050.459262:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232050.459276:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607037056/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232050.459283:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607036928:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:0.0:1761232050.459303:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232050.459308:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232050.459309:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232050.459315:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4922 00000100:00080000:0.0:1761232050.459318:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232050.459322:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232050.459327:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232050.459342:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607037120/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232050.459411:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232050/real 1761232050] req@0000000099514377 x1846780607036928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232075 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232050.459432:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607036928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232075 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232050.459444:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232050.459449:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232050.459454:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607036928:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232050.459463:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036992:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232050.459490:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232050/real 1761232050] req@000000000243251f x1846780607036992/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232075 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232050.459503:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607036992/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232075 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232050.459512:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232050.459514:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232050.459517:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607036992:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232050.459547:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037056:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232050.459618:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232050/real 1761232050] req@00000000271a9d8e x1846780607037056/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232075 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232050.459631:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607037056/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232075 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232050.459641:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232050.459644:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232050.459647:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037056:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232050.459654:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037120:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232050.459677:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232050/real 1761232050] req@0000000023ab3a3f x1846780607037120/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232075 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232050.459697:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607037120/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232075 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232050.459706:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232050.459708:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232050.459711:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037120:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232050.907114:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232051.931132:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232052.955103:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232053.979032:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232055.003104:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232055.579037:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232055.579051:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232055.579058:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232055.579071:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4927 00000100:00080000:3.0:1761232055.579076:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232055.579085:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232055.579130:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607037184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232055.579177:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232055.579182:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232055.579184:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232055.579189:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4927 00000100:00080000:3.0:1761232055.579191:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232055.579195:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232055.579207:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232055.579221:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607037248/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232055.579234:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232055.579239:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232055.579241:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232055.579249:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4927 00000100:00080000:3.0:1761232055.579253:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1761232055.579260:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607037184:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761232055.579263:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232055.579272:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232055.579288:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607037312/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232055.579300:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232055.579306:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232055.579307:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232055.579315:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4927 00000100:00080000:3.0:1761232055.579317:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232055.579320:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232055.579325:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232055.579337:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607037376/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761232055.579449:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037248:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761232055.579535:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037312:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761232055.579584:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037376:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232056.027162:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232057.051096:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232058.075191:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761232058.651333:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232055/real 1761232058] req@0000000083f16a6b x1846780607037248/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232080 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761232058.651380:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607037248/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232080 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761232058.651399:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761232058.651406:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761232058.651413:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037248:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761232058.651424:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232055/real 1761232058] req@0000000090b40c66 x1846780607037312/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232080 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761232058.651445:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607037312/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232080 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761232058.651460:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761232058.651500:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761232058.651507:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037312:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761232058.651518:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232055/real 1761232058] req@000000001f8e688e x1846780607037376/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232080 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761232058.651539:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607037376/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232080 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761232058.651552:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761232058.651556:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761232058.651560:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037376:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761232058.651569:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232055/real 1761232058] req@00000000a58ca11e x1846780607037184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232080 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761232058.651586:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607037184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232080 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761232058.651598:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761232058.651602:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761232058.651607:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607037184:192.168.201.104@tcp:250:kworker/u8:0.0 00010000:00010000:0.0:1761232059.099128:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232060.123016:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232060.698955:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232060.698970:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232060.698975:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232060.698988:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4933 00000100:00080000:3.0:1761232060.698993:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232060.699002:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232060.699043:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607037440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232060.699078:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232060.699083:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232060.699084:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232060.699089:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4933 00000100:00080000:3.0:1761232060.699092:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232060.699095:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232060.699104:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232060.699117:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607037504/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232060.699127:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232060.699131:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232060.699133:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232060.699137:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4933 00000100:00080000:3.0:1761232060.699140:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232060.699144:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232060.699149:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232060.699161:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607037568/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232060.699171:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232060.699174:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232060.699176:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232060.699180:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4933 00000100:00080000:3.0:1761232060.699183:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232060.699187:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232060.699193:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232060.699203:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607037440:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232060.699208:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607037632/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761232060.699343:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232060/real 1761232060] req@0000000031e043f4 x1846780607037440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232085 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761232060.699375:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607037440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232085 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761232060.699392:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761232060.699398:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761232060.699404:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607037440:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761232060.699429:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037504:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761232060.699509:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232060/real 1761232060] req@000000007361665d x1846780607037504/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232085 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761232060.699531:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607037504/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232085 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761232060.699546:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761232060.699550:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761232060.699555:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037504:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761232060.699566:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037568:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761232060.699610:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232060/real 1761232060] req@00000000f87d0618 x1846780607037568/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232085 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761232060.699643:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607037568/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232085 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761232060.699669:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761232060.699674:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761232060.699681:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037568:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761232060.699729:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037632:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761232060.699782:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232060/real 1761232060] req@00000000c7e4e432 x1846780607037632/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232085 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761232060.699803:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607037632/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232085 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761232060.699817:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761232060.699822:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761232060.699898:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037632:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232061.147177:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232062.171179:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232063.195063:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232064.218996:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232065.243078:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232065.818958:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232065.818974:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232065.818979:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232065.818994:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4938 00000100:00080000:3.0:1761232065.819000:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232065.819011:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232065.819063:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607037696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232065.819108:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232065.819115:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232065.819118:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232065.819125:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4938 00000100:00080000:3.0:1761232065.819154:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232065.819157:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232065.819165:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232065.819176:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607037760/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232065.819183:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232065.819185:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232065.819187:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232065.819190:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4938 00000100:00080000:3.0:1761232065.819192:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232065.819195:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232065.819199:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232065.819208:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607037824/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232065.819216:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232065.819218:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232065.819219:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232065.819225:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4938 00000100:00100000:0.0:1761232065.819226:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607037696:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761232065.819227:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232065.819233:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232065.819238:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232065.819245:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607037888/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761232065.819316:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232065/real 1761232065] req@00000000bd684ccc x1846780607037696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232090 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761232065.819330:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607037696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232090 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761232065.819339:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761232065.819341:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761232065.819344:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607037696:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761232065.819350:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037760:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761232065.819367:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232065/real 1761232065] req@0000000018a8f5da x1846780607037760/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232090 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761232065.819375:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607037760/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232090 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761232065.819380:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761232065.819381:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761232065.819383:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037760:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:0.0:1761232065.819444:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037824:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761232065.819463:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232065/real 1761232065] req@000000005a9bdefd x1846780607037824/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232090 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761232065.819471:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607037824/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232090 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761232065.819477:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761232065.819478:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761232065.819480:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037824:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761232065.819494:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037888:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:0.0:1761232065.819508:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232065/real 1761232065] req@0000000086ccb527 x1846780607037888/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232090 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:0.0:1761232065.819516:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607037888/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232090 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:0.0:1761232065.819521:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761232065.819523:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:0.0:1761232065.819524:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607037888:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232066.267124:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232067.291151:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232068.315146:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232069.339099:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232070.363170:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232070.939046:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232070.939059:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232070.939065:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232070.939077:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4943 00000100:00080000:0.0:1761232070.939082:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232070.939091:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232070.939152:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607037952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232070.939179:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232070.939182:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232070.939183:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232070.939186:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4943 00000100:00080000:0.0:1761232070.939188:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232070.939192:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232070.939200:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232070.939211:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607038016/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232070.939218:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607037952:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:0.0:1761232070.939222:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232070.939226:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232070.939226:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232070.939230:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4943 00000100:00080000:0.0:1761232070.939231:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232070.939233:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232070.939237:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232070.939249:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607038080/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232070.939272:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232070.939276:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232070.939278:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232070.939282:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4943 00000100:00080000:0.0:1761232070.939285:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232070.939289:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232070.939295:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232070.939309:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607038144/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232070.939387:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232070/real 1761232070] req@000000005a096598 x1846780607037952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232095 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232070.939403:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607037952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232095 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232070.939411:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232070.939414:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232070.939417:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607037952:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232070.939441:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038016:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232070.939463:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232070/real 1761232070] req@00000000275aeee5 x1846780607038016/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232095 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232070.939471:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607038016/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232095 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232070.939477:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232070.939478:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232070.939480:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038016:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232070.939484:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038080:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232070.939498:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232070/real 1761232070] req@000000002ef1f54f x1846780607038080/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232095 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232070.939504:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607038080/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232095 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232070.939511:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232070.939512:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232070.939514:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038080:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232070.939517:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038144:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232070.939531:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232070/real 1761232070] req@0000000023ab3a3f x1846780607038144/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232095 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232070.939537:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607038144/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232095 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232070.939543:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232070.939545:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232070.939547:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038144:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232071.387158:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232072.411105:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232073.435145:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232074.459190:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232075.483131:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232076.059063:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232076.059077:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232076.059083:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232076.059096:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4948 00000100:00080000:0.0:1761232076.059101:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232076.059109:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232076.059173:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607038208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232076.059216:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232076.059221:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232076.059224:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232076.059229:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4948 00000100:00080000:0.0:1761232076.059232:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232076.059238:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232076.059249:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232076.059262:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607038272/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232076.059312:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232076.059316:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232076.059318:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232076.059322:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4948 00000100:00080000:0.0:1761232076.059325:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232076.059328:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232076.059336:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232076.059348:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607038336/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232076.059359:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232076.059363:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232076.059366:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232076.059370:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4948 00000100:00080000:0.0:1761232076.059373:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232076.059376:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232076.059381:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232076.059385:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607038208:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:0.0:1761232076.059397:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607038400/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232076.059504:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232076/real 1761232076] req@00000000271a9d8e x1846780607038208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232101 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232076.059534:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607038208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232101 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232076.059547:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232076.059551:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232076.059556:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607038208:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232076.059569:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038272:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232076.059596:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232076/real 1761232076] req@000000000243251f x1846780607038272/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232101 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232076.059608:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607038272/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232101 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232076.059616:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232076.059619:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232076.059622:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038272:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232076.059628:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038336:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232076.059649:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232076/real 1761232076] req@0000000099514377 x1846780607038336/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232101 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232076.059660:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607038336/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232101 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232076.059670:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232076.059672:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232076.059675:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038336:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232076.059706:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038400:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232076.059728:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232076/real 1761232076] req@00000000fcce404c x1846780607038400/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232101 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232076.059739:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607038400/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232101 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232076.059749:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232076.059752:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232076.059755:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038400:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232076.507118:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232077.531172:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232078.554947:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232079.579132:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232080.603121:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232081.179084:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232081.179097:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232081.179102:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232081.179115:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4953 00000100:00080000:3.0:1761232081.179119:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232081.179127:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232081.179171:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607038464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232081.179242:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232081.179248:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232081.179250:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232081.179255:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4953 00000100:00080000:3.0:1761232081.179258:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232081.179262:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232081.179273:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232081.179286:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607038528/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232081.179297:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232081.179301:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232081.179303:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232081.179309:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4953 00000100:00080000:3.0:1761232081.179313:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232081.179317:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232081.179322:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232081.179334:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607038592/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232081.179345:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232081.179348:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232081.179351:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232081.179355:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4953 00000100:00080000:3.0:1761232081.179358:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1761232081.179365:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607038464:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761232081.179367:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232081.179378:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232081.179390:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607038656/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232081.179489:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232081/real 1761232081] req@00000000daa39bb1 x1846780607038464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232106 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232081.179511:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607038464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232106 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232081.179525:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232081.179529:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232081.179534:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607038464:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232081.179542:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038528:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232081.179568:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232081/real 1761232081] req@000000003ad149cc x1846780607038528/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232106 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232081.179580:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607038528/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232106 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232081.179589:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232081.179591:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232081.179594:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038528:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232081.179624:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038592:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232081.179649:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232081/real 1761232081] req@0000000088473b3e x1846780607038592/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232106 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232081.179661:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607038592/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232106 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232081.179671:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232081.179673:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232081.179676:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038592:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232081.179682:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038656:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232081.179702:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232081/real 1761232081] req@000000004e3f612a x1846780607038656/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232106 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232081.179714:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607038656/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232106 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232081.179723:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232081.179725:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232081.179728:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038656:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232081.627140:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232082.651130:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232083.675129:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232084.699155:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232085.723098:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232086.298987:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232086.299002:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232086.299007:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232086.299019:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4958 00000100:00080000:0.0:1761232086.299023:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232086.299032:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232086.299077:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607038720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232086.299114:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232086.299119:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232086.299121:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232086.299126:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4958 00000100:00080000:0.0:1761232086.299128:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232086.299132:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232086.299141:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232086.299180:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607038784/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232086.299196:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232086.299200:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232086.299202:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232086.299208:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4958 00000100:00080000:0.0:1761232086.299211:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232086.299217:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232086.299224:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232086.299239:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607038848/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232086.299254:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607038720:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761232086.299256:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232086.299270:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232086.299273:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232086.299282:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4958 00000100:00080000:0.0:1761232086.299287:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232086.299293:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232086.299304:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232086.299330:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607038912/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232086.299376:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232086/real 1761232086] req@000000007f60e210 x1846780607038720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232111 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232086.299400:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607038720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232111 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232086.299414:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232086.299419:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232086.299425:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607038720:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232086.299435:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038784:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232086.299465:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232086/real 1761232086] req@0000000001b2f813 x1846780607038784/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232111 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232086.299479:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607038784/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232111 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232086.299488:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232086.299490:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232086.299493:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038784:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232086.299523:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038848:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232086.299549:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232086/real 1761232086] req@00000000b8f2be78 x1846780607038848/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232111 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232086.299561:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607038848/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232111 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232086.299571:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232086.299573:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232086.299576:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038848:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232086.299582:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038912:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232086.299602:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232086/real 1761232086] req@00000000fcce404c x1846780607038912/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232111 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232086.299623:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607038912/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232111 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232086.299632:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232086.299634:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232086.299637:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607038912:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232086.747221:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232087.771204:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232088.795176:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232089.819213:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232090.843131:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232091.418999:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232091.419018:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232091.419024:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232091.419041:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4963 00000100:00080000:0.0:1761232091.419047:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232091.419056:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232091.419134:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607038976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232091.419176:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232091.419183:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232091.419186:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232091.419193:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4963 00000100:00080000:0.0:1761232091.419197:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232091.419203:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232091.419220:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232091.419245:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607039040/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232091.419266:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761232091.419267:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607038976:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:0.0:1761232091.419281:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232091.419287:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232091.419296:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4963 00000100:00080000:0.0:1761232091.419301:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232091.419308:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232091.419318:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232091.419337:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607039104/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232091.419349:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232091.419352:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232091.419354:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232091.419358:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4963 00000100:00080000:0.0:1761232091.419360:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232091.419366:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232091.419372:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232091.419394:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607039168/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232091.419394:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232091/real 1761232091] req@0000000099514377 x1846780607038976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232116 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232091.419449:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607038976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232116 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232091.419469:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232091.419475:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232091.419483:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607038976:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232091.419519:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039040:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232091.419576:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232091/real 1761232091] req@000000000243251f x1846780607039040/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232116 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232091.419600:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607039040/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232116 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232091.419615:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232091.419619:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232091.419625:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039040:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232091.419638:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039104:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232091.419688:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232091/real 1761232091] req@00000000271a9d8e x1846780607039104/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232116 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232091.419709:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607039104/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232116 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232091.419724:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232091.419729:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232091.419733:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039104:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232091.419740:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039168:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232091.419772:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232091/real 1761232091] req@0000000023ab3a3f x1846780607039168/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232116 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232091.419783:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607039168/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232116 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232091.419792:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232091.419794:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232091.419797:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039168:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232091.867012:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232092.891180:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232093.915151:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232094.939013:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232095.963144:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232096.539109:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232096.539122:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232096.539127:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232096.539140:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4968 00000100:00080000:3.0:1761232096.539145:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232096.539154:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232096.539203:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607039232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232096.539243:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232096.539250:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232096.539253:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232096.539260:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4968 00000100:00080000:3.0:1761232096.539264:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232096.539270:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232096.539284:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232096.539306:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607039296/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232096.539323:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232096.539329:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232096.539331:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232096.539337:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4968 00000100:00100000:1.0:1761232096.539340:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607039232:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761232096.539341:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232096.539347:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232096.539354:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232096.539371:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607039360/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232096.539386:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232096.539391:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232096.539393:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232096.539398:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4968 00000100:00080000:3.0:1761232096.539401:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232096.539407:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232096.539412:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232096.539428:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607039424/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232096.539464:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232096/real 1761232096] req@000000004e3f612a x1846780607039232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232121 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232096.539494:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607039232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232121 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232096.539513:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232096.539518:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232096.539525:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607039232:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232096.539537:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039296:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232096.539578:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232096/real 1761232096] req@0000000088473b3e x1846780607039296/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232121 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232096.539591:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607039296/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232121 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232096.539600:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232096.539603:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232096.539606:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039296:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232096.539638:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039360:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232096.539663:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232096/real 1761232096] req@000000003ad149cc x1846780607039360/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232121 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232096.539675:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607039360/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232121 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232096.539685:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232096.539687:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232096.539690:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039360:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232096.539696:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039424:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232096.539716:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232096/real 1761232096] req@00000000daa39bb1 x1846780607039424/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232121 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232096.539728:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607039424/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232121 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232096.539737:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232096.539739:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232096.539741:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039424:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232096.987099:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232098.011229:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232099.035162:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232100.059009:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232101.083181:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232101.659042:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232101.659053:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232101.659055:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232101.659064:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4974 00000100:00080000:3.0:1761232101.659066:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232101.659073:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232101.659102:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607039488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232101.659124:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232101.659126:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232101.659127:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232101.659130:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4974 00000100:00080000:3.0:1761232101.659131:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232101.659133:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232101.659140:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232101.659146:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607039552/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232101.659151:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232101.659153:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232101.659154:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232101.659158:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4974 00000100:00080000:3.0:1761232101.659159:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232101.659162:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232101.659165:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232101.659173:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607039616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232101.659179:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232101.659182:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232101.659183:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232101.659185:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607039488:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761232101.659189:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4974 00000100:00080000:3.0:1761232101.659193:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232101.659195:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232101.659197:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232101.659204:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607039680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232101.659263:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232101/real 1761232101] req@0000000086ccb527 x1846780607039488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232126 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232101.659275:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607039488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232126 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232101.659282:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232101.659284:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232101.659286:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607039488:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232101.659291:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039552:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232101.659305:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232101/real 1761232101] req@000000005a9bdefd x1846780607039552/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232126 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232101.659312:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607039552/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232126 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232101.659316:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232101.659318:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232101.659319:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039552:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232101.659337:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039616:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232101.659350:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232101/real 1761232101] req@0000000018a8f5da x1846780607039616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232126 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232101.659356:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607039616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232126 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232101.659361:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232101.659362:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232101.659364:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039616:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232101.659367:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039680:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232101.659378:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232101/real 1761232101] req@00000000bd684ccc x1846780607039680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232126 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232101.659384:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607039680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232126 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232101.659388:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232101.659389:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232101.659391:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039680:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232102.107150:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232103.131186:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232104.155173:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232105.179218:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232106.203138:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232106.779035:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232106.779052:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232106.779058:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232106.779074:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4979 00000100:00080000:3.0:1761232106.779079:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232106.779090:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232106.779152:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607039744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232106.779207:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232106.779214:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232106.779217:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232106.779226:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4979 00000100:00080000:3.0:1761232106.779231:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232106.779237:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232106.779254:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232106.779279:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607039808/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232106.779299:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232106.779306:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232106.779309:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232106.779317:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4979 00000100:00080000:3.0:1761232106.779322:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232106.779331:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1761232106.779336:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607039744:192.168.201.104@tcp:250:kworker/u8:0.0 00000008:00100000:3.0:1761232106.779343:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232106.779375:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607039872/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232106.779397:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232106.779404:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232106.779406:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232106.779412:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4979 00000100:00080000:3.0:1761232106.779417:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232106.779425:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232106.779435:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232106.779440:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232106/real 1761232106] req@00000000c7e4e432 x1846780607039744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232131 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232106.779461:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607039936/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232106.779465:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607039744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232131 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232106.779557:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232106.779563:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232106.779570:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607039744:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232106.779579:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039808:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232106.779598:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232106/real 1761232106] req@00000000f87d0618 x1846780607039808/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232131 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232106.779604:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607039808/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232131 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232106.779608:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232106.779610:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232106.779611:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039808:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232106.779637:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039872:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232106.779651:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232106/real 1761232106] req@000000007361665d x1846780607039872/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232131 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232106.779658:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607039872/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232131 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232106.779663:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232106.779664:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232106.779666:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039872:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232106.779670:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039936:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232106.779681:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232106/real 1761232106] req@0000000031e043f4 x1846780607039936/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232131 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232106.779687:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607039936/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232131 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232106.779692:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232106.779693:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232106.779694:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607039936:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232107.227266:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232108.251055:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232109.275222:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232110.299195:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232111.323212:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232111.898978:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232111.898994:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232111.898998:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232111.899011:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4984 00000100:00080000:3.0:1761232111.899016:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232111.899025:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232111.899070:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607040000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232111.899106:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232111.899111:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232111.899113:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232111.899118:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4984 00000100:00080000:3.0:1761232111.899120:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232111.899123:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232111.899133:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232111.899145:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607040064/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232111.899155:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232111.899158:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232111.899160:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232111.899165:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4984 00000100:00080000:3.0:1761232111.899168:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232111.899172:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232111.899177:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232111.899192:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607040128/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232111.899205:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232111.899210:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1761232111.899212:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607040000:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:3.0:1761232111.899214:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232111.899226:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4984 00000100:00080000:3.0:1761232111.899229:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232111.899232:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232111.899238:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232111.899252:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607040192/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232111.899333:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232111/real 1761232111] req@00000000a58ca11e x1846780607040000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232136 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232111.899356:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607040000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232136 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232111.899369:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232111.899373:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232111.899378:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607040000:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232111.899387:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040064:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232111.899417:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232111/real 1761232111] req@000000001f8e688e x1846780607040064/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232136 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232111.899430:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607040064/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232136 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232111.899447:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232111.899450:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232111.899453:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040064:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232111.899483:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040128:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232111.899509:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232111/real 1761232111] req@0000000090b40c66 x1846780607040128/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232136 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232111.899521:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607040128/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232136 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232111.899530:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232111.899532:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232111.899535:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040128:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232111.899541:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040192:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232111.899561:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232111/real 1761232111] req@0000000083f16a6b x1846780607040192/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232136 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232111.899572:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607040192/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232136 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232111.899581:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232111.899583:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232111.899586:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040192:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232112.347127:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232113.371052:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232114.395212:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232115.419161:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232116.443120:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232117.018980:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232117.018994:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232117.018999:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232117.019011:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4989 00000100:00080000:3.0:1761232117.019016:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232117.019024:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232117.019067:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607040256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232117.019101:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232117.019105:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232117.019107:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232117.019111:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4989 00000100:00080000:3.0:1761232117.019114:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232117.019117:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232117.019130:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232117.019142:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607040320/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232117.019153:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232117.019156:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232117.019158:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232117.019162:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4989 00000100:00080000:3.0:1761232117.019163:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232117.019166:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232117.019170:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232117.019182:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607040384/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232117.019201:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232117.019205:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232117.019207:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232117.019211:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4989 00000100:00080000:3.0:1761232117.019213:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232117.019217:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232117.019222:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232117.019233:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607040448/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232117.019250:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607040256:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232117.019364:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232117/real 1761232117] req@000000002ce934d3 x1846780607040256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232142 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232117.019386:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607040256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232142 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232117.019398:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232117.019401:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232117.019406:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607040256:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232117.019415:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040320:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232117.019442:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232117/real 1761232117] req@000000006572b591 x1846780607040320/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232142 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232117.019455:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607040320/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232142 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232117.019463:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232117.019465:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232117.019468:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040320:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232117.019474:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040384:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232117.019495:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232117/real 1761232117] req@00000000fc01c41c x1846780607040384/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232142 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232117.019506:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607040384/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232142 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232117.019515:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232117.019519:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232117.019521:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040384:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232117.019552:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040448:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232117.019576:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232117/real 1761232117] req@00000000408f088c x1846780607040448/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232142 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232117.019587:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607040448/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232142 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232117.019596:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232117.019598:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232117.019601:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040448:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232117.467138:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232118.491144:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232119.515191:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232120.539128:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232121.563054:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232122.138983:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232122.138994:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232122.138997:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232122.139008:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4994 00000100:00080000:3.0:1761232122.139028:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232122.139035:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232122.139078:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607040512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232122.139105:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232122.139109:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232122.139111:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232122.139116:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4994 00000100:00080000:3.0:1761232122.139118:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232122.139121:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232122.139129:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232122.139156:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607040576/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232122.139164:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232122.139167:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232122.139168:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232122.139171:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4994 00000100:00080000:3.0:1761232122.139173:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232122.139175:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232122.139178:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232122.139186:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607040640/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232122.139193:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232122.139197:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232122.139198:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232122.139201:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4994 00000100:00080000:3.0:1761232122.139204:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232122.139206:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232122.139210:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232122.139219:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607040704/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232122.139223:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607040512:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232122.139317:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040576:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232122.139338:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040640:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232122.139361:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040704:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232122.587103:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232123.611073:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232124.635105:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761232125.211214:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232122/real 1761232125] req@000000002ce934d3 x1846780607040704/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232147 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232125.211246:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607040704/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232147 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232125.211261:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232125.211266:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232125.211272:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040704:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232125.211280:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232122/real 1761232125] req@00000000408f088c x1846780607040512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232147 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232125.211294:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607040512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232147 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232125.211303:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232125.211306:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232125.211310:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607040512:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232125.211317:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232122/real 1761232125] req@00000000fc01c41c x1846780607040576/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232147 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232125.211331:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607040576/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232147 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232125.211338:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232125.211342:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232125.211345:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040576:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232125.211351:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232122/real 1761232125] req@000000006572b591 x1846780607040640/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232147 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232125.211363:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607040640/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232147 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232125.211372:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232125.211375:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232125.211378:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040640:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232125.659157:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232126.683085:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232127.259056:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232127.259072:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232127.259079:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232127.259095:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 4999 00000100:00080000:3.0:1761232127.259101:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232127.259110:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232127.259163:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607040768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232127.259203:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232127.259208:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232127.259210:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232127.259215:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4999 00000100:00080000:3.0:1761232127.259217:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232127.259220:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232127.259232:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232127.259244:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607040832/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232127.259255:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232127.259259:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232127.259260:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232127.259263:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4999 00000100:00080000:3.0:1761232127.259265:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232127.259268:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232127.259272:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232127.259282:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607040896/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232127.259290:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232127.259293:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232127.259294:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232127.259299:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 4999 00000100:00080000:3.0:1761232127.259301:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232127.259306:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232127.259310:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232127.259320:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607040960/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232127.259348:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607040768:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232127.259503:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232127/real 1761232127] req@0000000083f16a6b x1846780607040768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232152 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232127.259531:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607040768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232152 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232127.259547:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232127.259552:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232127.259559:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607040768:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232127.259570:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040832:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232127.259617:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232127/real 1761232127] req@0000000090b40c66 x1846780607040832/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232152 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232127.259636:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607040832/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232152 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232127.259649:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232127.259653:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232127.259658:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040832:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232127.259668:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040896:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232127.259740:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232127/real 1761232127] req@000000001f8e688e x1846780607040896/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232152 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232127.259759:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607040896/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232152 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232127.259782:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232127.259786:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232127.259790:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040896:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232127.259892:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040960:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232127.259943:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232127/real 1761232127] req@00000000a58ca11e x1846780607040960/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232152 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232127.259961:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607040960/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232152 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232127.259976:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232127.259980:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232127.259986:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607040960:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232127.707040:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232128.731142:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232129.755092:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232130.779037:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232131.803124:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232132.379025:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232132.379035:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232132.379039:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232132.379048:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5004 00000100:00080000:2.0:1761232132.379051:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232132.379057:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232132.379092:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607041024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232132.379119:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232132.379123:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232132.379125:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232132.379128:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5004 00000100:00080000:2.0:1761232132.379130:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232132.379132:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232132.379139:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232132.379148:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607041088/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232132.379154:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232132.379156:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232132.379157:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232132.379159:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5004 00000100:00080000:2.0:1761232132.379160:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232132.379163:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232132.379165:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232132.379173:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607041152/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232132.379178:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232132.379180:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232132.379180:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232132.379182:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5004 00000100:00080000:2.0:1761232132.379184:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232132.379186:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232132.379189:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232132.379194:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607041216/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232132.379278:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607041024:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232132.379443:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232132/real 1761232132] req@0000000017386078 x1846780607041024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232157 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232132.379467:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607041024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232157 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232132.379482:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232132.379486:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232132.379492:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607041024:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232132.379501:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041088:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232132.379534:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232132/real 1761232132] req@00000000f38f27df x1846780607041088/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232157 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232132.379547:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607041088/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232157 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232132.379555:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232132.379558:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232132.379561:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041088:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232132.379567:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041152:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232132.379590:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232132/real 1761232132] req@00000000c069b612 x1846780607041152/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232157 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232132.379601:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607041152/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232157 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232132.379611:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232132.379614:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232132.379617:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041152:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232132.379623:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041216:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232132.379678:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232132/real 1761232132] req@0000000064978143 x1846780607041216/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232157 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232132.379690:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607041216/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232157 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232132.379706:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232132.379712:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232132.379717:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041216:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232132.827189:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232133.851086:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232134.875188:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232135.899152:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232136.923055:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232137.499053:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232137.499067:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232137.499072:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232137.499084:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5009 00000100:00080000:0.0:1761232137.499089:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232137.499097:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232137.499143:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607041280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232137.499188:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232137.499193:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232137.499195:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232137.499201:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5009 00000100:00080000:0.0:1761232137.499203:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232137.499207:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232137.499219:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232137.499234:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607041344/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232137.499246:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232137.499250:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232137.499253:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232137.499257:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607041280:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761232137.499264:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5009 00000100:00080000:0.0:1761232137.499273:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232137.499277:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232137.499282:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232137.499299:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607041408/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232137.499309:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232137.499314:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232137.499316:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232137.499320:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5009 00000100:00080000:0.0:1761232137.499322:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232137.499327:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232137.499332:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232137.499344:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607041472/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232137.499385:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232137/real 1761232137] req@000000002ef1f54f x1846780607041280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232162 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232137.499408:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607041280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232162 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232137.499421:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232137.499425:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232137.499430:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607041280:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232137.499459:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041344:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232137.499489:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232137/real 1761232137] req@00000000275aeee5 x1846780607041344/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232162 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232137.499502:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607041344/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232162 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232137.499510:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232137.499512:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232137.499515:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041344:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232137.499522:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041408:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232137.499544:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232137/real 1761232137] req@000000005a096598 x1846780607041408/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232162 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232137.499564:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607041408/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232162 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232137.499604:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232137.499607:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232137.499611:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041408:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232137.499618:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041472:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232137.499647:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232137/real 1761232137] req@0000000023ab3a3f x1846780607041472/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232162 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232137.499659:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607041472/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232162 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232137.499669:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232137.499671:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232137.499674:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041472:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232137.947103:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232138.971126:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232139.995142:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232141.019100:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232142.043099:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232142.619028:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232142.619052:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232142.619057:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232142.619070:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5015 00000100:00080000:3.0:1761232142.619074:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232142.619085:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232142.619127:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607041536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232142.619164:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232142.619169:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232142.619171:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232142.619175:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5015 00000100:00080000:3.0:1761232142.619178:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232142.619182:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232142.619191:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232142.619203:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607041600/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232142.619214:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232142.619217:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232142.619219:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232142.619223:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5015 00000100:00080000:3.0:1761232142.619224:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232142.619227:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232142.619232:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232142.619243:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607041664/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232142.619252:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232142.619256:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232142.619257:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232142.619263:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5015 00000100:00080000:3.0:1761232142.619267:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232142.619271:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232142.619276:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232142.619287:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607041728/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232142.619307:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607041536:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232142.619418:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232142/real 1761232142] req@0000000031e043f4 x1846780607041536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232167 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232142.619439:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607041536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232167 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232142.619452:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232142.619456:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232142.619461:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607041536:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232142.619470:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041600:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232142.619497:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232142/real 1761232142] req@000000007361665d x1846780607041600/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232167 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232142.619509:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607041600/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232167 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232142.619559:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232142.619562:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232142.619566:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041600:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232142.619573:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041664:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232142.619603:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232142/real 1761232142] req@00000000f87d0618 x1846780607041664/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232167 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232142.619615:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607041664/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232167 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232142.619625:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232142.619628:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232142.619631:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041664:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232142.619663:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041728:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232142.619686:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232142/real 1761232142] req@00000000c7e4e432 x1846780607041728/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232167 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232142.619698:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607041728/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232167 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232142.619707:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232142.619710:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232142.619713:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041728:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232143.067078:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232144.091087:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232145.115089:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232146.139014:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232147.163127:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232147.738963:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232147.738976:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232147.738981:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232147.738994:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5020 00000100:00080000:3.0:1761232147.738998:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232147.739006:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232147.739050:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607041792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232147.739084:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232147.739088:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232147.739090:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232147.739095:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5020 00000100:00080000:3.0:1761232147.739098:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232147.739101:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232147.739111:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232147.739123:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607041856/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232147.739133:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232147.739138:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232147.739140:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232147.739143:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5020 00000100:00080000:3.0:1761232147.739146:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232147.739149:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232147.739153:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232147.739164:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607041920/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232147.739174:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232147.739178:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232147.739181:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232147.739186:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5020 00000100:00080000:3.0:1761232147.739189:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232147.739192:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232147.739199:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232147.739223:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607041984/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232147.739286:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607041792:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232147.739477:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232147/real 1761232147] req@00000000bd684ccc x1846780607041792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232172 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232147.739521:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607041792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232172 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232147.739536:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232147.739540:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232147.739562:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607041792:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232147.739572:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041856:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232147.739609:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232147/real 1761232147] req@0000000018a8f5da x1846780607041856/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232172 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232147.739622:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607041856/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232172 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232147.739647:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232147.739650:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232147.739653:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041856:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232147.739660:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041920:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232147.739687:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232147/real 1761232147] req@000000005a9bdefd x1846780607041920/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232172 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232147.739699:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607041920/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232172 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232147.739708:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232147.739710:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232147.739713:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041920:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232147.739718:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041984:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232147.739738:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232147/real 1761232147] req@0000000086ccb527 x1846780607041984/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232172 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232147.739749:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607041984/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232172 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232147.739758:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232147.739760:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232147.739762:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607041984:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232148.187236:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232149.211013:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232150.235127:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232151.259170:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232152.283116:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232152.859066:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232152.859080:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232152.859085:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232152.859099:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5025 00000100:00080000:2.0:1761232152.859103:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232152.859112:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232152.859159:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607042048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232152.859197:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232152.859202:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232152.859204:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232152.859209:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5025 00000100:00080000:2.0:1761232152.859211:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232152.859215:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232152.859224:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232152.859237:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607042112/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232152.859247:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232152.859252:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232152.859254:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232152.859258:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5025 00000100:00080000:2.0:1761232152.859261:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232152.859264:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232152.859268:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232152.859278:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607042176/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232152.859288:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232152.859292:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232152.859294:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232152.859298:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5025 00000100:00080000:2.0:1761232152.859301:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232152.859306:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232152.859311:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232152.859321:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607042240/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232152.859347:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607042048:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232152.859510:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232152/real 1761232152] req@0000000091106a97 x1846780607042048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232177 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232152.859532:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607042048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232177 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232152.859544:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232152.859549:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232152.859553:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607042048:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232152.859562:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042112:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232152.859589:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232152/real 1761232152] req@0000000094135f44 x1846780607042112/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232177 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232152.859602:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607042112/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232177 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232152.859610:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232152.859612:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232152.859615:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042112:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232152.859621:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042176:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232152.859642:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232152/real 1761232152] req@0000000064978143 x1846780607042176/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232177 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232152.859653:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607042176/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232177 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232152.859663:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232152.859665:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232152.859668:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042176:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232152.859703:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042240:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232152.859726:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232152/real 1761232152] req@00000000c069b612 x1846780607042240/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232177 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232152.859747:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607042240/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232177 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232152.859756:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232152.859758:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232152.859761:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042240:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232153.307073:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232154.331121:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232155.355242:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232156.379181:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232157.403158:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232157.978955:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232157.978967:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232157.978971:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232157.978982:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5030 00000100:00080000:3.0:1761232157.978985:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232157.978993:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232157.979028:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607042304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232157.979057:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232157.979060:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232157.979062:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232157.979065:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5030 00000100:00080000:3.0:1761232157.979067:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232157.979070:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232157.979079:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232157.979093:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607042368/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232157.979127:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232157.979134:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1761232157.979135:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607042304:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761232157.979137:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232157.979149:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5030 00000100:00080000:3.0:1761232157.979151:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232157.979156:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232157.979164:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232157.979186:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607042432/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232157.979199:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232157.979204:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232157.979205:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232157.979210:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5030 00000100:00080000:3.0:1761232157.979213:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232157.979220:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232157.979228:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232157.979234:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232157/real 1761232157] req@00000000daa39bb1 x1846780607042304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232182 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232157.979246:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607042496/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232157.979254:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607042304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232182 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232157.979270:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232157.979273:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232157.979277:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607042304:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232157.979300:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042368:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232157.979323:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232157/real 1761232157] req@000000003ad149cc x1846780607042368/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232182 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232157.979331:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607042368/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232182 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232157.979357:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232157.979362:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232157.979366:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042368:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232157.979392:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042432:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232157.979438:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232157/real 1761232157] req@0000000088473b3e x1846780607042432/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232182 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232157.979459:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607042432/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232182 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232157.979475:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232157.979479:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232157.979483:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042432:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232157.979491:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042496:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232157.979522:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232157/real 1761232157] req@000000004e3f612a x1846780607042496/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232182 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232157.979534:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607042496/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232182 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232157.979543:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232157.979546:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232157.979549:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042496:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232158.427146:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232159.451176:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232160.475155:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232161.499125:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232162.523128:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232163.098974:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232163.098988:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232163.098993:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232163.099005:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5035 00000100:00080000:3.0:1761232163.099010:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232163.099019:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232163.099067:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607042560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232163.099101:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232163.099106:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232163.099108:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232163.099113:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5035 00000100:00080000:3.0:1761232163.099116:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232163.099119:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232163.099129:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232163.099141:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607042624/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232163.099151:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232163.099154:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232163.099156:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232163.099161:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5035 00000100:00080000:3.0:1761232163.099164:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232163.099167:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232163.099172:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232163.099182:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607042688/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232163.099191:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232163.099194:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232163.099196:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232163.099201:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5035 00000100:00080000:3.0:1761232163.099204:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232163.099207:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232163.099212:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232163.099222:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607042752/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232163.099252:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607042560:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232163.099410:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232163/real 1761232163] req@000000004e3f612a x1846780607042560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232188 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232163.099432:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607042560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232188 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232163.099446:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232163.099450:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232163.099454:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607042560:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232163.099463:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042624:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232163.099494:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232163/real 1761232163] req@0000000088473b3e x1846780607042624/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232188 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232163.099507:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607042624/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232188 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232163.099514:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232163.099517:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232163.099520:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042624:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232163.099526:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042688:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232163.099547:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232163/real 1761232163] req@000000003ad149cc x1846780607042688/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232188 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232163.099558:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607042688/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232188 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232163.099567:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232163.099570:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232163.099573:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042688:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232163.099578:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042752:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232163.099599:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232163/real 1761232163] req@00000000daa39bb1 x1846780607042752/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232188 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232163.099611:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607042752/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232188 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232163.099621:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232163.099623:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232163.099625:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042752:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232163.547121:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232164.571142:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232165.595143:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232166.619102:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232167.643151:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232168.219062:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232168.219079:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232168.219085:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232168.219101:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5040 00000100:00080000:1.0:1761232168.219107:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232168.219118:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232168.219173:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607042816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232168.219244:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232168.219252:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232168.219256:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232168.219266:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5040 00000100:00080000:1.0:1761232168.219271:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232168.219280:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1761232168.219289:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607042816:192.168.201.104@tcp:250:kworker/u8:0.0 00000008:00100000:1.0:1761232168.219296:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232168.219327:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607042880/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232168.219349:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232168.219355:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232168.219358:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232168.219365:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5040 00000100:00080000:1.0:1761232168.219370:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232168.219377:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232168.219387:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232168.219404:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232168/real 1761232168] req@00000000d07717c4 x1846780607042816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232193 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232168.219415:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607042944/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232168.219434:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607042816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232193 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232168.219440:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232168.219458:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232168.219463:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232168.219464:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232168.219471:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00080000:1.0:1761232168.219479:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5040 00000100:00100000:3.0:1761232168.219481:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607042816:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:1.0:1761232168.219485:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232168.219499:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232168.219508:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232168.219528:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607043008/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232168.219550:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042880:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232168.219593:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232168/real 1761232168] req@00000000db10af3c x1846780607042880/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232193 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232168.219608:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607042880/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232193 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232168.219617:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232168.219620:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232168.219624:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042880:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232168.219630:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042944:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232168.219652:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232168/real 1761232168] req@00000000659d78e3 x1846780607042944/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232193 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232168.219664:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607042944/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232193 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232168.219674:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232168.219676:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232168.219678:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607042944:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232168.219693:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043008:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232168.219714:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232168/real 1761232168] req@00000000b72feabb x1846780607043008/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232193 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232168.219725:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607043008/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232193 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232168.219734:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232168.219737:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232168.219739:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043008:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232168.667118:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232169.691100:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232170.715095:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232171.739110:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232172.763088:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232173.339028:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232173.339043:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232173.339048:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232173.339061:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5045 00000100:00080000:2.0:1761232173.339066:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232173.339075:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232173.339124:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607043072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232173.339160:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232173.339205:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232173.339207:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232173.339212:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5045 00000100:00080000:2.0:1761232173.339214:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232173.339217:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232173.339227:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232173.339241:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607043136/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232173.339253:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232173.339256:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232173.339258:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232173.339265:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5045 00000100:00100000:3.0:1761232173.339266:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607043072:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:2.0:1761232173.339268:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232173.339275:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232173.339284:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232173.339297:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607043200/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232173.339305:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232173.339309:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232173.339310:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232173.339313:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5045 00000100:00080000:2.0:1761232173.339316:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232173.339319:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232173.339322:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232173.339333:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607043264/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232173.339394:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232173/real 1761232173] req@00000000f38f27df x1846780607043072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232198 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232173.339419:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607043072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232198 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232173.339433:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232173.339437:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232173.339459:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607043072:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232173.339494:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043136:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232173.339554:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232173/real 1761232173] req@0000000017386078 x1846780607043136/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232198 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232173.339575:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607043136/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232198 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232173.339590:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232173.339594:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232173.339600:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043136:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232173.339610:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043200:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232173.339645:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232173/real 1761232173] req@00000000c069b612 x1846780607043200/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232198 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232173.339680:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607043200/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232198 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232173.339693:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232173.339697:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232173.339701:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043200:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232173.339709:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043264:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232173.339745:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232173/real 1761232173] req@0000000064978143 x1846780607043264/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232198 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232173.339761:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607043264/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232198 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232173.339772:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232173.339775:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232173.339780:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043264:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232173.787248:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232174.811245:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232175.835196:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232176.859237:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232177.883146:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232178.459092:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232178.459156:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232178.459163:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232178.459180:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5050 00000100:00080000:3.0:1761232178.459186:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232178.459198:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232178.459252:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607043328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232178.459303:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232178.459308:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232178.459310:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232178.459316:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5050 00000100:00080000:3.0:1761232178.459319:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232178.459323:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232178.459333:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232178.459351:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607043392/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232178.459363:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232178.459367:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232178.459368:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232178.459372:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5050 00000100:00080000:3.0:1761232178.459374:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232178.459383:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1761232178.459385:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607043328:192.168.201.104@tcp:250:kworker/u8:0.0 00000008:00100000:3.0:1761232178.459391:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232178.459405:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607043456/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232178.459416:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232178.459420:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232178.459421:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232178.459425:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5050 00000100:00080000:3.0:1761232178.459427:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232178.459431:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232178.459437:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232178.459449:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607043520/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232178.459511:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232178/real 1761232178] req@0000000086ccb527 x1846780607043328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232203 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232178.459533:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607043328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232203 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232178.459546:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232178.459550:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232178.459555:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607043328:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232178.459563:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043392:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232178.459592:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232178/real 1761232178] req@000000005a9bdefd x1846780607043392/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232203 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232178.459604:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607043392/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232203 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232178.459612:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232178.459614:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232178.459617:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043392:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232178.459666:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043456:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232178.459696:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232178/real 1761232178] req@0000000018a8f5da x1846780607043456/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232203 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232178.459717:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607043456/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232203 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232178.459727:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232178.459730:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232178.459733:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043456:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232178.459739:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043520:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232178.459764:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232178/real 1761232178] req@00000000bd684ccc x1846780607043520/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232203 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232178.459775:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607043520/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232203 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232178.459784:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232178.459786:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232178.459789:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043520:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232178.907151:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232179.931058:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232180.955181:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232181.979061:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232183.003096:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232183.579096:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232183.579110:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232183.579115:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232183.579129:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5055 00000100:00080000:0.0:1761232183.579133:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232183.579141:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232183.579189:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607043584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232183.579226:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232183.579231:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232183.579233:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232183.579238:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5055 00000100:00080000:0.0:1761232183.579240:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232183.579244:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232183.579253:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232183.579266:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607043648/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232183.579279:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232183.579284:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232183.579286:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232183.579290:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5055 00000100:00080000:0.0:1761232183.579292:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232183.579296:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232183.579302:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232183.579313:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607043712/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232183.579323:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232183.579327:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232183.579329:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232183.579333:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5055 00000100:00080000:0.0:1761232183.579336:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232183.579339:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232183.579344:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232183.579355:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607043776/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232183.579375:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607043584:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232183.579491:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232183/real 1761232183] req@00000000271a9d8e x1846780607043584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232208 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232183.579512:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607043584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232208 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232183.579525:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232183.579529:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232183.579533:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607043584:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232183.579541:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043648:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232183.579568:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232183/real 1761232183] req@000000000243251f x1846780607043648/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232208 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232183.579581:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607043648/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232208 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232183.579589:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232183.579593:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232183.579596:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043648:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232183.579602:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043712:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232183.579625:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232183/real 1761232183] req@0000000099514377 x1846780607043712/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232208 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232183.579636:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607043712/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232208 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232183.579646:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232183.579650:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232183.579655:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043712:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232183.579698:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043776:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232183.579751:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232183/real 1761232183] req@00000000fcce404c x1846780607043776/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232208 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232183.579773:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607043776/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232208 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232183.579789:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232183.579794:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232183.579799:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043776:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232184.027140:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232185.051219:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232186.075231:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232187.099183:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232188.123080:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232188.698949:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232188.698964:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232188.698969:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232188.698980:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5061 00000100:00080000:0.0:1761232188.698985:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232188.699036:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232188.699079:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607043840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232188.699113:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232188.699118:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232188.699120:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232188.699125:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5061 00000100:00080000:0.0:1761232188.699128:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232188.699131:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232188.699141:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232188.699153:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607043904/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232188.699164:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232188.699168:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232188.699171:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232188.699175:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5061 00000100:00080000:0.0:1761232188.699177:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232188.699182:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232188.699187:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232188.699199:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607043968/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232188.699212:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232188.699216:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232188.699220:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232188.699222:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607043840:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761232188.699229:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5061 00000100:00080000:0.0:1761232188.699236:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232188.699241:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232188.699246:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232188.699264:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607044032/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232188.699347:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043904:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232188.699396:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043968:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232188.699426:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044032:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232189.147130:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232190.171111:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232191.195024:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761232191.771393:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232188/real 1761232191] req@000000007f60e210 x1846780607043968/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232213 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232191.771430:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607043968/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232213 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232191.771457:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232191.771462:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232191.771467:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043968:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232191.771476:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232188/real 1761232191] req@00000000fcce404c x1846780607044032/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232213 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232191.771492:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607044032/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232213 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232191.771503:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232191.771507:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232191.771512:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044032:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232191.771520:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232188/real 1761232191] req@00000000b8f2be78 x1846780607043840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232213 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232191.771535:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607043840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232213 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232191.771546:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232191.771551:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232191.771555:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607043840:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232191.771563:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232188/real 1761232191] req@0000000001b2f813 x1846780607043904/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232213 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232191.771578:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607043904/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232213 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232191.771586:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232191.771589:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232191.771593:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607043904:192.168.201.104@tcp:38:kworker/u8:2.0 00010000:00010000:0.0:1761232192.219140:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232193.243044:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232193.819117:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232193.819126:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232193.819129:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232193.819138:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5066 00000100:00080000:2.0:1761232193.819142:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232193.819148:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232193.819178:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607044096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232193.819208:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232193.819213:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232193.819221:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232193.819226:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5066 00000100:00080000:2.0:1761232193.819229:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232193.819232:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232193.819243:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232193.819259:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607044160/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232193.819265:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607044096:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761232193.819280:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232193.819284:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232193.819287:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232193.819292:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5066 00000100:00080000:2.0:1761232193.819295:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232193.819298:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232193.819304:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232193.819318:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607044224/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232193.819328:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232193.819332:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232193.819333:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232193.819337:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5066 00000100:00080000:2.0:1761232193.819340:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232193.819345:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232193.819350:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232193.819358:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232193/real 1761232193] req@0000000094135f44 x1846780607044096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232218 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761232193.819366:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607044288/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232193.819390:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607044096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232218 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232193.819411:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232193.819415:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232193.819421:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607044096:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232193.819448:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044160:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232193.819483:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232193/real 1761232193] req@0000000091106a97 x1846780607044160/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232218 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232193.819493:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607044160/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232218 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232193.819500:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232193.819502:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232193.819504:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044160:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232193.819509:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044224:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232193.819534:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232193/real 1761232193] req@0000000064978143 x1846780607044224/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232218 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232193.819545:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607044224/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232218 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232193.819554:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232193.819557:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232193.819560:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044224:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232193.819567:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044288:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232193.819593:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232193/real 1761232193] req@00000000c069b612 x1846780607044288/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232218 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232193.819614:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607044288/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232218 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232193.819624:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232193.819627:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232193.819630:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044288:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232194.267223:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232195.291107:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232196.315082:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232197.339131:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232198.363049:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232198.938988:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232198.939001:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232198.939005:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232198.939018:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5071 00000100:00080000:2.0:1761232198.939023:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232198.939031:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232198.939077:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607044352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232198.939111:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232198.939117:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232198.939119:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232198.939123:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5071 00000100:00080000:2.0:1761232198.939126:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232198.939129:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232198.939139:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232198.939154:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607044416/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232198.939166:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232198.939170:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232198.939173:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232198.939177:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5071 00000100:00080000:2.0:1761232198.939180:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232198.939183:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232198.939188:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232198.939200:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607044480/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232198.939211:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232198.939216:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232198.939218:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232198.939227:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5071 00000100:00100000:1.0:1761232198.939228:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607044352:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:2.0:1761232198.939231:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232198.939239:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232198.939252:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232198.939267:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607044544/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232198.939369:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232198/real 1761232198] req@0000000017386078 x1846780607044352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232223 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232198.939392:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607044352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232223 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232198.939404:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232198.939408:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232198.939413:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607044352:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232198.939421:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044416:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232198.939451:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232198/real 1761232198] req@00000000f38f27df x1846780607044416/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232223 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232198.939464:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607044416/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232223 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232198.939472:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232198.939475:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232198.939478:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044416:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232198.939507:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044480:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232198.939532:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232198/real 1761232198] req@00000000c069b612 x1846780607044480/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232223 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232198.939545:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607044480/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232223 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232198.939554:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232198.939557:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232198.939560:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044480:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232198.939566:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044544:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232198.939600:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232198/real 1761232198] req@0000000064978143 x1846780607044544/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232223 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232198.939617:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607044544/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232223 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232198.939628:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232198.939701:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232198.939706:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044544:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232199.387072:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232200.411116:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232201.435179:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232202.459177:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232203.483182:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232204.059027:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232204.059040:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232204.059045:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232204.059057:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5076 00000100:00080000:1.0:1761232204.059061:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232204.059070:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232204.059122:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607044608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232204.059168:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232204.059173:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232204.059175:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232204.059180:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5076 00000100:00080000:1.0:1761232204.059183:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232204.059188:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232204.059200:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232204.059216:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607044672/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232204.059226:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232204.059229:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232204.059231:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232204.059234:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5076 00000100:00080000:1.0:1761232204.059236:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232204.059239:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232204.059244:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232204.059257:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607044736/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232204.059267:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232204.059271:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232204.059274:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232204.059278:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5076 00000100:00080000:1.0:1761232204.059280:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232204.059283:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232204.059287:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232204.059291:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607044608:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232204.059301:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607044800/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232204.059398:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232204/real 1761232204] req@00000000b72feabb x1846780607044608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232229 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232204.059419:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607044608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232229 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232204.059431:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232204.059435:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232204.059439:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607044608:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232204.059447:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044672:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232204.059472:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232204/real 1761232204] req@00000000659d78e3 x1846780607044672/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232229 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232204.059483:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607044672/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232229 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232204.059491:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232204.059493:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232204.059496:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044672:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232204.059501:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044736:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232204.059525:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232204/real 1761232204] req@00000000db10af3c x1846780607044736/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232229 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232204.059536:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607044736/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232229 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232204.059545:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232204.059546:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232204.059549:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044736:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232204.059577:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044800:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232204.059598:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232204/real 1761232204] req@00000000d07717c4 x1846780607044800/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232229 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232204.059608:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607044800/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232229 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232204.059616:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232204.059618:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232204.059621:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044800:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232204.507114:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232205.531122:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232206.555157:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232207.579086:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232208.603139:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232209.179034:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232209.179046:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232209.179050:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232209.179062:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5081 00000100:00080000:2.0:1761232209.179066:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232209.179074:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232209.179118:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607044864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232209.179149:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232209.179153:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232209.179155:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232209.179159:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5081 00000100:00080000:2.0:1761232209.179161:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232209.179164:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232209.179173:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232209.179185:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607044928/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232209.179194:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232209.179197:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232209.179199:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232209.179202:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5081 00000100:00080000:2.0:1761232209.179203:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232209.179206:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232209.179211:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232209.179220:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607044992/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232209.179229:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232209.179233:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232209.179235:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232209.179238:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5081 00000100:00080000:2.0:1761232209.179240:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232209.179243:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232209.179246:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232209.179256:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607045056/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232209.179264:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607044864:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232209.179366:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232209/real 1761232209] req@0000000091106a97 x1846780607044864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232234 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232209.179385:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607044864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232234 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232209.179396:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232209.179399:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232209.179404:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607044864:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232209.179411:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044928:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232209.179433:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232209/real 1761232209] req@0000000094135f44 x1846780607044928/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232234 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232209.179444:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607044928/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232234 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232209.179451:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232209.179453:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232209.179455:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044928:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232209.179460:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044992:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232209.179478:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232209/real 1761232209] req@0000000064978143 x1846780607044992/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232234 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232209.179488:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607044992/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232234 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232209.179496:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232209.179498:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232209.179500:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607044992:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232209.179527:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045056:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232209.179547:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232209/real 1761232209] req@00000000c069b612 x1846780607045056/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232234 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232209.179564:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607045056/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232234 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232209.179572:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232209.179573:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232209.179576:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045056:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232209.627030:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232210.651108:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232211.675105:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232212.699077:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232213.723153:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232214.299068:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232214.299082:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232214.299087:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232214.299101:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5086 00000100:00080000:1.0:1761232214.299105:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232214.299114:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232214.299161:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607045120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232214.299198:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232214.299202:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232214.299204:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232214.299209:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5086 00000100:00080000:1.0:1761232214.299212:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232214.299215:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232214.299225:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232214.299238:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607045184/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232214.299248:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232214.299252:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232214.299254:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232214.299257:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5086 00000100:00080000:1.0:1761232214.299259:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232214.299262:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232214.299266:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232214.299276:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607045248/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232214.299287:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232214.299293:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232214.299295:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232214.299299:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5086 00000100:00080000:1.0:1761232214.299301:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232214.299306:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232214.299310:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232214.299324:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607045120:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232214.299326:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607045312/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232214.299441:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232214/real 1761232214] req@00000000d07717c4 x1846780607045120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232239 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232214.299462:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607045120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232239 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232214.299475:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232214.299479:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232214.299483:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607045120:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232214.299492:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045184:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232214.299518:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232214/real 1761232214] req@00000000db10af3c x1846780607045184/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232239 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232214.299531:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607045184/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232239 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232214.299540:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232214.299542:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232214.299545:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045184:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232214.299552:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045248:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232214.299573:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232214/real 1761232214] req@00000000659d78e3 x1846780607045248/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232239 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232214.299584:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607045248/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232239 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232214.299593:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232214.299596:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232214.299599:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045248:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232214.299629:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045312:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232214.299652:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232214/real 1761232214] req@00000000b72feabb x1846780607045312/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232239 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232214.299664:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607045312/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232239 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232214.299672:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232214.299674:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232214.299677:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045312:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232214.747156:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232215.771138:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232216.795075:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232217.819160:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232218.843158:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232219.419076:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232219.419089:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232219.419095:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232219.419107:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5091 00000100:00080000:0.0:1761232219.419112:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232219.419120:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232219.419174:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607045376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232219.419201:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232219.419205:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232219.419207:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232219.419211:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5091 00000100:00080000:0.0:1761232219.419214:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232219.419218:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232219.419227:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232219.419241:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607045440/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232219.419252:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761232219.419256:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607045376:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:0.0:1761232219.419257:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232219.419259:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232219.419262:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5091 00000100:00080000:0.0:1761232219.419264:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232219.419268:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232219.419273:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232219.419284:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607045504/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232219.419293:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232219.419296:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232219.419297:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232219.419300:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5091 00000100:00080000:0.0:1761232219.419302:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232219.419306:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232219.419310:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232219.419318:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607045568/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232219.419347:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232219/real 1761232219] req@0000000099514377 x1846780607045376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232244 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232219.419367:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607045376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232244 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232219.419379:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232219.419383:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232219.419387:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607045376:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232219.419410:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045440:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232219.419434:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232219/real 1761232219] req@000000000243251f x1846780607045440/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232244 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232219.419443:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607045440/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232244 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232219.419448:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232219.419451:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232219.419453:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045440:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232219.419456:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045504:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232219.419471:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232219/real 1761232219] req@00000000271a9d8e x1846780607045504/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232244 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232219.419478:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607045504/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232244 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232219.419503:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232219.419505:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232219.419508:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045504:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232219.419514:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045568:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232219.419537:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232219/real 1761232219] req@0000000023ab3a3f x1846780607045568/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232244 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232219.419548:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607045568/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232244 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232219.419557:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232219.419559:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232219.419562:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045568:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232219.867085:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232220.891108:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232221.915111:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232222.939066:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232223.963126:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232224.539005:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232224.539022:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232224.539028:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232224.539044:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5096 00000100:00080000:1.0:1761232224.539050:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232224.539060:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232224.539107:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607045632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232224.539138:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232224.539143:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232224.539145:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232224.539149:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5096 00000100:00080000:1.0:1761232224.539151:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232224.539156:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232224.539175:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232224.539177:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607045632:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232224.539198:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607045696/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232224.539212:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232224.539216:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232224.539219:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232224.539222:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5096 00000100:00080000:1.0:1761232224.539225:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232224.539228:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232224.539233:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232224.539244:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607045760/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232224.539253:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232224.539255:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232224.539257:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232224.539261:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5096 00000100:00080000:1.0:1761232224.539264:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232224.539269:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232224.539275:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232224.539286:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232224/real 1761232224] req@00000000b72feabb x1846780607045632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232249 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232224.539292:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607045824/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232224.539351:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607045632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232249 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232224.539365:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232224.539368:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232224.539372:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607045632:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232224.539418:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045696:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232224.539453:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232224/real 1761232224] req@00000000659d78e3 x1846780607045696/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232249 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232224.539466:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607045696/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232249 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232224.539475:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232224.539479:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232224.539482:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045696:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232224.539489:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045760:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232224.539511:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232224/real 1761232224] req@00000000db10af3c x1846780607045760/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232249 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232224.539522:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607045760/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232249 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232224.539531:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232224.539534:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232224.539537:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045760:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232224.539542:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045824:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232224.539562:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232224/real 1761232224] req@00000000d07717c4 x1846780607045824/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232249 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232224.539573:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607045824/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232249 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232224.539637:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232224.539640:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232224.539644:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045824:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232224.987028:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232226.011139:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232227.035069:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232228.059143:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232229.083050:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232229.658975:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232229.658989:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232229.658994:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232229.659006:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5102 00000100:00080000:1.0:1761232229.659011:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232229.659021:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232229.659069:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607045888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232229.659105:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232229.659109:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232229.659111:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232229.659116:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5102 00000100:00080000:1.0:1761232229.659118:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232229.659122:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232229.659132:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232229.659145:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607045952/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232229.659155:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232229.659159:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232229.659160:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232229.659164:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5102 00000100:00080000:1.0:1761232229.659166:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232229.659169:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232229.659173:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232229.659182:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607046016/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232229.659193:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232229.659196:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232229.659198:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232229.659203:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5102 00000100:00080000:1.0:1761232229.659205:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232229.659209:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232229.659215:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232229.659229:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607045888:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232229.659230:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607046080/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232229.659346:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232229/real 1761232229] req@00000000d07717c4 x1846780607045888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232254 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232229.659377:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607045888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232254 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232229.659389:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232229.659392:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232229.659397:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607045888:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232229.659406:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045952:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232229.659432:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232229/real 1761232229] req@00000000db10af3c x1846780607045952/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232254 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232229.659444:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607045952/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232254 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232229.659452:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232229.659455:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232229.659458:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607045952:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232229.659463:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046016:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232229.659485:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232229/real 1761232229] req@00000000659d78e3 x1846780607046016/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232254 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232229.659496:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607046016/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232254 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232229.659505:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232229.659507:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232229.659510:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046016:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232229.659592:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046080:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232229.659626:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232229/real 1761232229] req@00000000b72feabb x1846780607046080/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232254 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232229.659666:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607046080/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232254 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232229.659681:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232229.659686:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232229.659691:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046080:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232230.107126:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232231.131183:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232232.155174:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232233.179124:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232234.203020:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232234.779103:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232234.779116:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232234.779120:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232234.779133:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5107 00000100:00080000:3.0:1761232234.779138:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232234.779147:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232234.779193:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607046144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232234.779233:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232234.779238:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232234.779240:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232234.779245:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5107 00000100:00080000:3.0:1761232234.779248:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232234.779252:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232234.779262:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232234.779279:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607046208/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232234.779292:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232234.779297:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232234.779299:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232234.779302:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607046144:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761232234.779309:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5107 00000100:00080000:3.0:1761232234.779316:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232234.779320:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232234.779326:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232234.779343:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607046272/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232234.779353:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232234.779358:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232234.779360:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232234.779364:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5107 00000100:00080000:3.0:1761232234.779366:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232234.779370:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232234.779375:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232234.779388:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607046336/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232234.779432:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232234/real 1761232234] req@00000000c7e4e432 x1846780607046144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232259 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232234.779455:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607046144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232259 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232234.779649:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232234.779654:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232234.779659:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607046144:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232234.779690:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046208:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232234.779727:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232234/real 1761232234] req@00000000f87d0618 x1846780607046208/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232259 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232234.779741:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607046208/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232259 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232234.779749:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232234.779752:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232234.779755:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046208:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232234.779763:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046272:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232234.779786:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232234/real 1761232234] req@000000007361665d x1846780607046272/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232259 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232234.779797:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607046272/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232259 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232234.779806:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232234.779809:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232234.779811:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046272:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232234.779817:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046336:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232234.779880:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232234/real 1761232234] req@0000000031e043f4 x1846780607046336/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232259 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232234.779893:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607046336/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232259 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232234.779902:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232234.779905:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232234.779908:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046336:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232235.227126:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232236.251082:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232237.275037:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232238.299126:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232239.323167:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232239.899029:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232239.899057:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232239.899064:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232239.899081:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5112 00000100:00080000:3.0:1761232239.899088:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232239.899099:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232239.899162:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607046400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232239.899204:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232239.899209:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232239.899211:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232239.899217:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5112 00000100:00080000:3.0:1761232239.899220:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232239.899223:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232239.899236:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232239.899250:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607046464/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232239.899261:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232239.899264:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232239.899266:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232239.899270:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5112 00000100:00080000:3.0:1761232239.899272:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232239.899275:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232239.899280:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232239.899293:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607046528/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232239.899303:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232239.899307:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232239.899311:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232239.899316:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5112 00000100:00080000:3.0:1761232239.899320:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232239.899324:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232239.899329:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232239.899342:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607046592/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232239.899376:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607046400:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232239.899526:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232239/real 1761232239] req@00000000a58ca11e x1846780607046400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232264 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232239.899548:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607046400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232264 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232239.899563:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232239.899568:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232239.899572:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607046400:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232239.899581:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046464:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232239.899608:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232239/real 1761232239] req@000000001f8e688e x1846780607046464/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232264 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232239.899621:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607046464/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232264 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232239.899629:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232239.899632:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232239.899635:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046464:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232239.899641:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046528:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232239.899663:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232239/real 1761232239] req@0000000090b40c66 x1846780607046528/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232264 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232239.899675:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607046528/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232264 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232239.899683:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232239.899686:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232239.899689:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046528:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232239.899723:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046592:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232239.899748:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232239/real 1761232239] req@0000000083f16a6b x1846780607046592/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232264 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232239.899760:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607046592/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232264 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232239.899769:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232239.899771:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232239.899774:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046592:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232240.347194:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232241.371149:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232242.395149:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232243.419148:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232244.443167:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232245.018951:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232245.018964:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232245.018968:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232245.018981:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5117 00000100:00080000:3.0:1761232245.018985:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232245.018993:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232245.019036:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607046656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232245.019068:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232245.019072:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232245.019074:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232245.019079:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5117 00000100:00080000:3.0:1761232245.019081:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232245.019084:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232245.019093:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232245.019104:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607046720/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232245.019114:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232245.019118:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232245.019121:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232245.019125:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5117 00000100:00080000:3.0:1761232245.019127:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232245.019131:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232245.019136:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232245.019149:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607046784/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232245.019160:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232245.019164:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232245.019165:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232245.019168:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5117 00000100:00080000:3.0:1761232245.019171:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232245.019174:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232245.019178:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232245.019192:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607046848/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232245.019193:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607046656:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232245.019323:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232245/real 1761232245] req@000000006572b591 x1846780607046656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232270 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232245.019344:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607046656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232270 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232245.019379:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232245.019383:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232245.019388:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607046656:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232245.019397:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046720:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232245.019426:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232245/real 1761232245] req@00000000fc01c41c x1846780607046720/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232270 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232245.019438:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607046720/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232270 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232245.019446:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232245.019449:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232245.019452:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046720:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232245.019480:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046784:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232245.019503:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232245/real 1761232245] req@00000000408f088c x1846780607046784/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232270 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232245.019514:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607046784/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232270 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232245.019522:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232245.019524:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232245.019527:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046784:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232245.019532:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046848:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232245.019550:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232245/real 1761232245] req@000000002ce934d3 x1846780607046848/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232270 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232245.019561:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607046848/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232270 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232245.019569:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232245.019571:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232245.019574:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046848:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232245.467123:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232246.491207:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232247.515092:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232248.539121:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232249.563102:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232250.139049:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232250.139063:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232250.139067:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232250.139079:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5122 00000100:00080000:2.0:1761232250.139084:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232250.139093:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232250.139138:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607046912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232250.139172:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232250.139178:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232250.139179:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232250.139184:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5122 00000100:00080000:2.0:1761232250.139186:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232250.139189:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232250.139198:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232250.139209:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607046976/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232250.139219:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232250.139222:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232250.139224:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232250.139227:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5122 00000100:00080000:2.0:1761232250.139229:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232250.139232:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232250.139236:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232250.139245:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607047040/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232250.139253:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232250.139257:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232250.139259:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232250.139263:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5122 00000100:00080000:2.0:1761232250.139265:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232250.139269:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232250.139273:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232250.139285:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607047104/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232250.139320:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607046912:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232250.139426:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232250/real 1761232250] req@00000000f38f27df x1846780607046912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232275 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232250.139453:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607046912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232275 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232250.139464:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232250.139468:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232250.139472:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607046912:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232250.139479:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046976:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232250.139504:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232250/real 1761232250] req@0000000017386078 x1846780607046976/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232275 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232250.139515:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607046976/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232275 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232250.139523:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232250.139525:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232250.139527:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607046976:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232250.139533:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047040:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232250.139553:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232250/real 1761232250] req@00000000c069b612 x1846780607047040/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232275 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232250.139564:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607047040/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232275 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232250.139573:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232250.139575:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232250.139577:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047040:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232250.139606:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047104:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232250.139627:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232250/real 1761232250] req@0000000064978143 x1846780607047104/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232275 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232250.139639:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607047104/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232275 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232250.139647:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232250.139649:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232250.139651:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047104:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232250.587054:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232251.611188:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232252.635181:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232253.659127:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232254.683127:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232255.259081:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232255.259106:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232255.259111:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232255.259123:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5127 00000100:00080000:1.0:1761232255.259127:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232255.259136:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232255.259184:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607047168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232255.259224:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232255.259228:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232255.259267:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232255.259281:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5127 00000100:00080000:1.0:1761232255.259283:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232255.259288:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232255.259298:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232255.259313:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607047232/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232255.259325:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232255.259330:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232255.259332:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232255.259337:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5127 00000100:00080000:1.0:1761232255.259340:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232255.259344:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232255.259348:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232255.259358:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607047296/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232255.259369:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761232255.259375:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607047168:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761232255.259376:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232255.259380:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232255.259391:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5127 00000100:00080000:1.0:1761232255.259396:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232255.259399:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232255.259405:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232255.259416:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607047360/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232255.259523:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047232:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232255.259598:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047296:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232255.259652:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047360:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232255.707113:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232256.731147:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232257.755149:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761232258.331438:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232255/real 1761232258] req@00000000db10af3c x1846780607047296/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232280 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232258.331484:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607047296/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232280 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232258.331504:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232258.331511:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232258.331518:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047296:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232258.331529:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232255/real 1761232258] req@00000000d07717c4 x1846780607047360/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232280 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232258.331548:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607047360/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232280 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232258.331562:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232258.331583:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232258.331587:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047360:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232258.331594:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232255/real 1761232258] req@00000000b72feabb x1846780607047168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232280 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232258.331610:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607047168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232280 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232258.331621:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232258.331624:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232258.331628:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607047168:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232258.331637:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232255/real 1761232258] req@00000000659d78e3 x1846780607047232/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232280 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232258.331659:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607047232/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232280 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232258.331669:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232258.331672:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232258.331676:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047232:192.168.201.104@tcp:38:kworker/u8:2.0 00010000:00010000:0.0:1761232258.779116:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232259.803109:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232260.378987:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232260.378997:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232260.379001:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232260.379011:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5132 00000100:00080000:1.0:1761232260.379015:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232260.379022:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232260.379054:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607047424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232260.379077:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232260.379081:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232260.379082:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232260.379086:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5132 00000100:00080000:1.0:1761232260.379087:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232260.379090:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232260.379097:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232260.379106:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607047488/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232260.379113:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232260.379115:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232260.379116:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232260.379119:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5132 00000100:00080000:1.0:1761232260.379120:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232260.379122:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232260.379125:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232260.379131:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607047552/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232260.379137:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232260.379139:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232260.379141:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232260.379144:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5132 00000100:00080000:1.0:1761232260.379146:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232260.379148:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232260.379151:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232260.379157:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607047616/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232260.379222:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607047424:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232260.379322:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232260/real 1761232260] req@00000000659d78e3 x1846780607047424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232285 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232260.379342:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607047424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232285 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232260.379354:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232260.379358:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232260.379362:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607047424:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232260.379371:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047488:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232260.379407:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232260/real 1761232260] req@00000000b72feabb x1846780607047488/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232285 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232260.379422:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607047488/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232285 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232260.379432:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232260.379435:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232260.379439:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047488:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232260.379447:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047552:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232260.379480:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232260/real 1761232260] req@00000000d07717c4 x1846780607047552/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232285 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232260.379493:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607047552/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232285 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232260.379504:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232260.379506:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232260.379509:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047552:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232260.379515:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047616:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232260.379544:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232260/real 1761232260] req@00000000db10af3c x1846780607047616/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232285 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232260.379556:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607047616/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232285 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232260.379566:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232260.379568:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232260.379572:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047616:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232260.827118:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232261.851132:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232262.875023:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232263.899205:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232264.922988:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232265.499047:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232265.499059:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232265.499065:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232265.499084:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5137 00000100:00080000:3.0:1761232265.499089:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232265.499105:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232265.499210:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607047680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232265.499246:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232265.499251:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232265.499253:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232265.499257:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5137 00000100:00080000:3.0:1761232265.499260:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232265.499265:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232265.499276:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232265.499291:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607047744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232265.499303:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232265.499307:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232265.499309:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232265.499313:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5137 00000100:00080000:3.0:1761232265.499315:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232265.499323:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1761232265.499327:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607047680:192.168.201.104@tcp:250:kworker/u8:0.0 00000008:00100000:3.0:1761232265.499331:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232265.499350:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607047808/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232265.499362:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232265.499367:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232265.499369:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232265.499374:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5137 00000100:00080000:3.0:1761232265.499376:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232265.499379:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232265.499384:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232265.499405:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607047872/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232265.499473:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232265/real 1761232265] req@000000002ce934d3 x1846780607047680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232290 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232265.499504:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607047680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232290 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232265.499521:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232265.499526:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232265.499533:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607047680:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232265.499546:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047744:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232265.499585:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232265/real 1761232265] req@00000000408f088c x1846780607047744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232290 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232265.499604:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607047744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232290 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232265.499618:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232265.499622:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232265.499627:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047744:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232265.499678:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047808:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232265.499724:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232265/real 1761232265] req@00000000fc01c41c x1846780607047808/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232290 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232265.499744:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607047808/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232290 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232265.499760:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232265.499763:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232265.499768:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047808:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232265.499778:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047872:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232265.499809:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232265/real 1761232265] req@000000006572b591 x1846780607047872/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232290 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232265.499865:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607047872/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232290 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232265.499881:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232265.499885:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232265.499890:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607047872:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232265.947135:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232266.971093:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232267.995219:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232269.019153:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232270.043183:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232270.619109:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232270.619122:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232270.619126:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232270.619138:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5143 00000100:00080000:1.0:1761232270.619142:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232270.619151:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232270.619196:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607047936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232270.619231:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232270.619235:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232270.619237:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232270.619241:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5143 00000100:00080000:1.0:1761232270.619244:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232270.619246:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232270.619256:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232270.619268:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607048000/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232270.619279:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232270.619282:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232270.619283:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232270.619287:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5143 00000100:00080000:1.0:1761232270.619291:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232270.619294:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232270.619299:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232270.619307:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607048064/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232270.619321:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607047936:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761232270.619324:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232270.619331:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232270.619336:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232270.619363:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5143 00000100:00080000:1.0:1761232270.619365:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232270.619369:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232270.619376:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232270.619392:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607048128/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232270.619496:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232270/real 1761232270] req@00000000db10af3c x1846780607047936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232295 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232270.619519:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607047936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232295 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232270.619533:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232270.619537:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232270.619542:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607047936:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232270.619551:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048000:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232270.619577:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232270/real 1761232270] req@00000000d07717c4 x1846780607048000/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232295 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232270.619588:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607048000/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232295 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232270.619596:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232270.619598:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232270.619601:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048000:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232270.619637:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048064:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232270.619660:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232270/real 1761232270] req@00000000b72feabb x1846780607048064/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232295 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232270.619680:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607048064/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232295 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232270.619688:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232270.619690:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232270.619693:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048064:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232270.619698:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048128:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232270.619716:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232270/real 1761232270] req@00000000659d78e3 x1846780607048128/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232295 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232270.619727:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607048128/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232295 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232270.619734:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232270.619736:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232270.619739:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048128:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232271.067148:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232272.091083:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232273.115125:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232274.139124:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232275.163143:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232275.739102:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232275.739115:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232275.739120:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232275.739132:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5148 00000100:00080000:2.0:1761232275.739136:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232275.739145:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232275.739191:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607048192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232275.739228:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232275.739233:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232275.739235:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232275.739239:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5148 00000100:00080000:2.0:1761232275.739242:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232275.739245:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232275.739271:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232275.739286:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607048256/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232275.739297:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232275.739301:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232275.739303:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232275.739306:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5148 00000100:00080000:2.0:1761232275.739308:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232275.739311:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232275.739318:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232275.739329:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607048320/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232275.739341:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232275.739346:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232275.739350:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232275.739354:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5148 00000100:00080000:2.0:1761232275.739357:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232275.739360:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232275.739366:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232275.739378:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607048384/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232275.739422:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607048192:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232275.739613:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232275/real 1761232275] req@0000000094135f44 x1846780607048192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232300 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232275.739637:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607048192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232300 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232275.739654:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232275.739659:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232275.739664:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607048192:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232275.739683:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048256:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232275.739712:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232275/real 1761232275] req@0000000091106a97 x1846780607048256/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232300 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232275.739725:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607048256/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232300 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232275.739733:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232275.739736:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232275.739739:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048256:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:3.0:1761232275.739745:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048320:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232275.739771:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232275/real 1761232275] req@0000000064978143 x1846780607048320/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232300 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232275.739783:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607048320/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232300 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232275.739792:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232275.739794:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232275.739797:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048320:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232275.739803:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048384:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:3.0:1761232275.739824:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232275/real 1761232275] req@00000000c069b612 x1846780607048384/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232300 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:3.0:1761232275.739896:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607048384/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232300 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:3.0:1761232275.739906:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232275.739909:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232275.739913:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048384:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232276.187194:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232277.211082:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232278.235103:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232279.259258:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232280.283229:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232280.859101:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232280.859124:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232280.859130:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232280.859148:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5153 00000100:00080000:3.0:1761232280.859154:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232280.859166:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232280.859234:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607048448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232280.859277:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232280.859282:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232280.859284:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232280.859289:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5153 00000100:00080000:3.0:1761232280.859291:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232280.859295:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232280.859308:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232280.859328:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607048512/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232280.859342:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232280.859347:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232280.859349:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232280.859353:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5153 00000100:00080000:3.0:1761232280.859354:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232280.859359:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232280.859364:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232280.859371:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607048448:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232280.859402:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607048576/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232280.859422:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232280.859427:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232280.859429:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232280.859433:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5153 00000100:00080000:3.0:1761232280.859436:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232280.859441:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232280.859447:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232280.859462:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607048640/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232280.859523:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232280/real 1761232280] req@0000000083f16a6b x1846780607048448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232305 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232280.859546:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607048448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232305 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232280.859559:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232280.859564:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232280.859569:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607048448:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232280.859579:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048512:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232280.859610:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232280/real 1761232280] req@0000000090b40c66 x1846780607048512/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232305 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232280.859623:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607048512/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232305 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232280.859631:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232280.859635:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232280.859638:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048512:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232280.859671:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048576:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232280.859696:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232280/real 1761232280] req@000000001f8e688e x1846780607048576/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232305 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232280.859708:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607048576/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232305 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232280.859718:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232280.859720:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232280.859723:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048576:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232280.859729:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048640:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232280.859754:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232280/real 1761232280] req@00000000a58ca11e x1846780607048640/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232305 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232280.859775:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607048640/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232305 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232280.859789:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232280.859793:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232280.859798:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048640:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232281.307188:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232282.331018:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232283.355268:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232284.379204:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232285.403222:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232285.979037:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232285.979048:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232285.979051:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232285.979062:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5158 00000100:00080000:2.0:1761232285.979065:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232285.979073:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232285.979110:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607048704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232285.979140:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232285.979143:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232285.979144:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232285.979148:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5158 00000100:00080000:2.0:1761232285.979151:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232285.979159:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232285.979168:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232285.979180:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607048768/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232285.979189:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232285.979192:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232285.979194:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232285.979197:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5158 00000100:00080000:2.0:1761232285.979199:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232285.979203:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232285.979206:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232285.979215:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607048832/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232285.979222:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607048704:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:2.0:1761232285.979226:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232285.979232:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232285.979233:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232285.979237:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5158 00000100:00080000:2.0:1761232285.979239:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232285.979242:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232285.979246:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232285.979258:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607048896/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232285.979347:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232285/real 1761232285] req@0000000017386078 x1846780607048704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232310 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232285.979369:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607048704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232310 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232285.979383:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232285.979387:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232285.979391:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607048704:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232285.979400:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048768:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232285.979429:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232285/real 1761232285] req@00000000f38f27df x1846780607048768/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232310 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232285.979441:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607048768/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232310 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232285.979450:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232285.979452:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232285.979455:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048768:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232285.979483:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048832:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232285.979508:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232285/real 1761232285] req@00000000c069b612 x1846780607048832/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232310 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232285.979520:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607048832/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232310 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232285.979529:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232285.979531:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232285.979534:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048832:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232285.979541:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048896:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232285.979561:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232285/real 1761232285] req@0000000064978143 x1846780607048896/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232310 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232285.979572:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607048896/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232310 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232285.979581:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232285.979583:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232285.979586:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607048896:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232286.427101:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232287.451196:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232288.475206:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232289.499170:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232290.523123:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232291.098976:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232291.098989:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232291.098994:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232291.099006:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5163 00000100:00080000:3.0:1761232291.099010:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232291.099019:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232291.099059:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607048960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232291.099090:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232291.099094:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232291.099096:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232291.099100:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5163 00000100:00080000:3.0:1761232291.099103:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232291.099106:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232291.099115:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232291.099127:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607049024/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232291.099137:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232291.099141:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232291.099143:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232291.099149:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5163 00000100:00080000:3.0:1761232291.099151:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232291.099155:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232291.099160:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232291.099174:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607048960:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:3.0:1761232291.099178:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607049088/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232291.099195:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232291.099198:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232291.099199:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232291.099204:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5163 00000100:00080000:3.0:1761232291.099207:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232291.099211:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232291.099215:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232291.099228:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607049152/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232291.099291:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232291/real 1761232291] req@0000000031e043f4 x1846780607048960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232316 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232291.099312:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607048960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232316 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232291.099323:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232291.099327:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232291.099331:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607048960:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232291.099339:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049024:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232291.099366:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232291/real 1761232291] req@000000007361665d x1846780607049024/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232316 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232291.099378:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607049024/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232316 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232291.099386:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232291.099389:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232291.099392:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049024:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232291.099438:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049088:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232291.099466:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232291/real 1761232291] req@00000000f87d0618 x1846780607049088/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232316 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232291.099478:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607049088/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232316 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232291.099488:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232291.099490:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232291.099493:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049088:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232291.099499:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049152:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232291.099518:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232291/real 1761232291] req@00000000c7e4e432 x1846780607049152/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232316 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232291.099530:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607049152/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232316 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232291.099538:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232291.099541:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232291.099543:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049152:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232291.547142:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232292.571100:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232293.595160:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232294.619138:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232295.643152:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232296.218975:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232296.218985:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232296.218988:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232296.218998:0:11821:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5168 00000100:00080000:2.0:1761232296.219001:0:11821:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232296.219007:0:11821:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232296.219048:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607049216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232296.219081:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232296.219086:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232296.219087:0:11821:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232296.219092:0:11821:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5168 00000100:00080000:2.0:1761232296.219094:0:11821:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232296.219097:0:11821:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232296.219107:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232296.219131:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607049280/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232296.219141:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232296.219144:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232296.219146:0:11821:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232296.219150:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5168 00000100:00080000:2.0:1761232296.219151:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232296.219155:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232296.219160:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232296.219171:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607049344/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232296.219184:0:11821:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232296.219188:0:11821:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232296.219190:0:11821:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232296.219191:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607049216:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00080000:2.0:1761232296.219197:0:11821:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5168 00000100:00080000:2.0:1761232296.219202:0:11821:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232296.219207:0:11821:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232296.219212:0:11821:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232296.219243:0:11821:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607049408/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232296.219313:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232296/real 1761232296] req@0000000091106a97 x1846780607049216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232321 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232296.219342:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607049216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232321 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232296.219353:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232296.219357:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232296.219361:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607049216:192.168.201.104@tcp:250:kworker/u8:0.0 00000100:00100000:1.0:1761232296.219368:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049280:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232296.219394:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232296/real 1761232296] req@0000000094135f44 x1846780607049280/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232321 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232296.219405:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607049280/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232321 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232296.219413:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232296.219415:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232296.219418:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049280:192.168.201.104@tcp:38:kworker/u8:0.0 00000100:00100000:1.0:1761232296.219444:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049344:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232296.219467:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232296/real 1761232296] req@0000000064978143 x1846780607049344/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232321 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232296.219478:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607049344/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232321 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232296.219486:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232296.219488:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232296.219491:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049344:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232296.219497:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049408:192.168.201.104@tcp:8:kworker/u8:0.0 00000100:00100000:1.0:1761232296.219516:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232296/real 1761232296] req@00000000c069b612 x1846780607049408/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232321 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00100000:1.0:1761232296.219527:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607049408/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232321 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:0.0' 00000100:00080000:1.0:1761232296.219535:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232296.219537:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232296.219539:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049408:192.168.201.104@tcp:8:kworker/u8:0.0 00010000:00010000:0.0:1761232296.667152:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232297.691130:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232298.715131:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232299.739100:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232300.763206:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232301.339060:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232301.339073:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232301.339078:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232301.339093:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5173 00000100:00080000:1.0:1761232301.339098:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232301.339107:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232301.339156:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607049472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232301.339209:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232301.339214:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232301.339216:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232301.339221:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5173 00000100:00080000:1.0:1761232301.339224:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232301.339227:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232301.339238:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232301.339252:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607049536/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232301.339262:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232301.339273:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232301.339275:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232301.339279:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5173 00000100:00080000:1.0:1761232301.339281:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232301.339285:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232301.339289:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232301.339298:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607049600/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232301.339308:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232301.339312:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232301.339314:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232301.339318:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5173 00000100:00080000:1.0:1761232301.339320:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232301.339324:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232301.339328:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232301.339342:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607049664/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232301.339407:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607049472:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232301.339570:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232301/real 1761232301] req@00000000659d78e3 x1846780607049472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232326 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232301.339595:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607049472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232326 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232301.339610:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232301.339615:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232301.339620:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607049472:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232301.339629:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049536:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232301.339659:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232301/real 1761232301] req@00000000b72feabb x1846780607049536/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232326 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232301.339672:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607049536/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232326 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232301.339680:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232301.339683:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232301.339685:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049536:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232301.339693:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049600:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232301.339763:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232301/real 1761232301] req@00000000d07717c4 x1846780607049600/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232326 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232301.339776:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607049600/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232326 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232301.339786:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232301.339789:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232301.339792:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049600:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232301.339799:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049664:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232301.339822:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232301/real 1761232301] req@00000000db10af3c x1846780607049664/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232326 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232301.339899:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607049664/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232326 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232301.339909:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232301.339912:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232301.339915:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049664:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232301.787193:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232302.811158:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232303.835061:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232304.859207:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232305.883170:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232306.458970:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232306.458982:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232306.458986:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232306.458997:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5178 00000100:00080000:1.0:1761232306.459002:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232306.459010:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232306.459051:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607049728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232306.459080:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232306.459083:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232306.459085:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232306.459089:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5178 00000100:00080000:1.0:1761232306.459091:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232306.459093:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232306.459101:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232306.459111:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607049792/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232306.459120:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232306.459123:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232306.459124:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232306.459127:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5178 00000100:00080000:1.0:1761232306.459129:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232306.459132:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232306.459137:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232306.459148:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607049856/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232306.459162:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232306.459167:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232306.459170:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232306.459177:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5178 00000100:00080000:1.0:1761232306.459180:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232306.459186:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1761232306.459192:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607049728:192.168.201.104@tcp:250:kworker/u8:2.0 00000008:00100000:1.0:1761232306.459194:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232306.459220:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607049920/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232306.459298:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232306/real 1761232306] req@00000000db10af3c x1846780607049728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232331 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232306.459316:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607049728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232331 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232306.459327:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232306.459330:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232306.459334:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607049728:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232306.459340:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049792:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232306.459362:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232306/real 1761232306] req@00000000d07717c4 x1846780607049792/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232331 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232306.459372:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607049792/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232331 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232306.459379:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232306.459381:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232306.459383:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049792:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232306.459388:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049856:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232306.459405:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232306/real 1761232306] req@00000000b72feabb x1846780607049856/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232331 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232306.459414:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607049856/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232331 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232306.459421:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232306.459423:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232306.459426:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049856:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232306.459451:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049920:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232306.459470:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232306/real 1761232306] req@00000000659d78e3 x1846780607049920/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232331 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232306.459480:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607049920/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232331 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232306.459487:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232306.459489:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232306.459491:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607049920:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232306.907149:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232307.931183:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232308.955328:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232309.979101:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232311.003092:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232311.579053:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232311.579067:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232311.579072:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232311.579085:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5183 00000100:00080000:2.0:1761232311.579089:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232311.579098:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232311.579145:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607049984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232311.579183:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232311.579188:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232311.579190:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232311.579194:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5183 00000100:00080000:2.0:1761232311.579197:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232311.579200:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232311.579213:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232311.579227:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607050048/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232311.579237:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232311.579241:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232311.579242:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232311.579246:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5183 00000100:00080000:2.0:1761232311.579248:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232311.579251:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232311.579255:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232311.579265:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607050112/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232311.579275:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232311.579279:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232311.579281:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232311.579287:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5183 00000100:00080000:2.0:1761232311.579289:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232311.579293:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232311.579298:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232311.579309:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607050176/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232311.579322:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607049984:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232311.579432:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232311/real 1761232311] req@00000000f38f27df x1846780607049984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232336 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232311.579453:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607049984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232336 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232311.579466:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232311.579470:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232311.579475:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607049984:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232311.579482:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050048:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232311.579509:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232311/real 1761232311] req@0000000017386078 x1846780607050048/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232336 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232311.579521:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607050048/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232336 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232311.579529:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232311.579532:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232311.579535:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050048:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232311.579541:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050112:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232311.579563:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232311/real 1761232311] req@00000000c069b612 x1846780607050112/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232336 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232311.579574:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607050112/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232336 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232311.579583:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232311.579585:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232311.579588:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050112:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232311.579684:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050176:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232311.579725:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232311/real 1761232311] req@0000000064978143 x1846780607050176/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232336 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232311.579743:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607050176/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232336 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232311.579754:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232311.579756:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232311.579759:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050176:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232312.027142:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232313.051131:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232314.075157:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232315.099150:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232316.123171:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232316.699037:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232316.699052:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232316.699057:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232316.699070:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5189 00000100:00080000:1.0:1761232316.699075:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232316.699084:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232316.699132:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607050240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232316.699170:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232316.699175:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232316.699177:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232316.699182:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5189 00000100:00080000:1.0:1761232316.699184:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232316.699188:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232316.699198:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232316.699211:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607050304/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232316.699221:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232316.699226:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232316.699229:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232316.699235:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5189 00000100:00080000:1.0:1761232316.699237:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232316.699242:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232316.699246:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232316.699257:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607050368/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232316.699266:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232316.699269:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232316.699271:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232316.699275:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5189 00000100:00080000:1.0:1761232316.699278:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232316.699281:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232316.699287:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232316.699293:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607050240:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232316.699303:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607050432/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232316.699425:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232316/real 1761232316] req@00000000659d78e3 x1846780607050240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232341 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232316.699470:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607050240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232341 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232316.699490:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232316.699495:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232316.699502:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607050240:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232316.699514:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050304:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232316.699595:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232316/real 1761232316] req@00000000b72feabb x1846780607050304/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232341 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232316.699618:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607050304/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232341 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232316.699632:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232316.699637:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232316.699643:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050304:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232316.699654:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050368:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232316.699705:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232316/real 1761232316] req@00000000d07717c4 x1846780607050368/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232341 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232316.699727:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607050368/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232341 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232316.699743:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232316.699747:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232316.699753:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050368:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232316.699798:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050432:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232316.699902:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232316/real 1761232316] req@00000000db10af3c x1846780607050432/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232341 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232316.699926:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607050432/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232341 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232316.699942:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232316.699946:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232316.699952:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050432:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232317.147109:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232318.171089:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232319.195136:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232320.219081:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232321.243080:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232321.819183:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232321.819200:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232321.819205:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232321.819222:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5194 00000100:00080000:3.0:1761232321.819228:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232321.819239:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232321.819303:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607050496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232321.819348:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232321.819353:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232321.819355:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232321.819361:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5194 00000100:00080000:3.0:1761232321.819363:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232321.819368:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232321.819382:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232321.819396:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607050560/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232321.819406:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232321.819410:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232321.819412:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232321.819416:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5194 00000100:00080000:3.0:1761232321.819419:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232321.819423:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232321.819428:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232321.819439:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607050624/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232321.819449:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232321.819454:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232321.819456:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232321.819461:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5194 00000100:00080000:3.0:1761232321.819464:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232321.819467:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232321.819499:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232321.819500:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607050496:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232321.819518:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607050688/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232321.819652:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050560:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232321.819682:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050624:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232321.819723:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050688:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232322.267101:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232323.291141:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232324.315057:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761232324.891172:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232321/real 1761232324] req@0000000086ccb527 x1846780607050688/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232346 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232324.891206:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607050688/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232346 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232324.891218:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232324.891223:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232324.891227:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050688:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232324.891233:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232321/real 1761232324] req@00000000bd684ccc x1846780607050496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232346 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232324.891244:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607050496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232346 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232324.891250:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232324.891252:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232324.891255:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607050496:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232324.891259:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232321/real 1761232324] req@0000000018a8f5da x1846780607050560/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232346 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232324.891269:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607050560/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232346 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232324.891276:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232324.891278:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232324.891281:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050560:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232324.891286:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232321/real 1761232324] req@000000005a9bdefd x1846780607050624/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232346 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232324.891303:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607050624/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232346 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232324.891310:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232324.891312:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232324.891315:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050624:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232325.339029:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232326.363052:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232326.938982:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232326.938998:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232326.939002:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232326.939016:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5199 00000100:00080000:3.0:1761232326.939021:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232326.939030:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232326.939076:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607050752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232326.939111:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232326.939116:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232326.939118:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232326.939123:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5199 00000100:00080000:3.0:1761232326.939125:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232326.939129:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232326.939138:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232326.939151:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607050816/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232326.939162:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232326.939167:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232326.939169:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232326.939174:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5199 00000100:00080000:3.0:1761232326.939176:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232326.939181:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232326.939186:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232326.939198:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607050752:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232326.939205:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607050880/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232326.939221:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232326.939224:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232326.939226:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232326.939231:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5199 00000100:00080000:3.0:1761232326.939234:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232326.939238:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232326.939243:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232326.939257:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607050944/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232326.939324:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232326/real 1761232326] req@00000000daa39bb1 x1846780607050752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232351 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232326.939356:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607050752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232351 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232326.939373:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232326.939378:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232326.939384:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607050752:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232326.939396:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050816:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232326.939474:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232326/real 1761232326] req@000000003ad149cc x1846780607050816/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232351 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232326.939495:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607050816/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232351 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232326.939509:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232326.939513:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232326.939519:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050816:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232326.939561:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050880:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232326.939614:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232326/real 1761232326] req@0000000088473b3e x1846780607050880/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232351 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232326.939635:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607050880/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232351 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232326.939651:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232326.939656:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232326.939661:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050880:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232326.939671:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050944:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232326.939717:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232326/real 1761232326] req@000000004e3f612a x1846780607050944/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232351 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232326.939738:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607050944/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232351 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232326.939754:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232326.939758:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232326.939763:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607050944:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232327.387097:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232328.411140:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232329.435082:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232330.459146:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232331.483150:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232332.059056:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232332.059074:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232332.059080:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232332.059097:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5204 00000100:00080000:1.0:1761232332.059102:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232332.059115:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232332.059174:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607051008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232332.059225:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232332.059232:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232332.059236:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232332.059244:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5204 00000100:00080000:1.0:1761232332.059248:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232332.059257:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232332.059271:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232332.059303:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607051072/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232332.059303:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607051008:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761232332.059336:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232332.059343:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232332.059347:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232332.059383:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5204 00000100:00080000:1.0:1761232332.059389:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232332.059397:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232332.059407:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232332.059431:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607051136/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232332.059448:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232332.059455:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232332.059459:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232332.059468:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5204 00000100:00080000:1.0:1761232332.059473:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232332.059481:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232332.059490:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232332.059498:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232332/real 1761232332] req@00000000db10af3c x1846780607051008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232357 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232332.059514:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607051200/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232332.059551:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607051008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232357 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232332.059567:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232332.059571:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232332.059579:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607051008:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232332.059611:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051072:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232332.059647:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232332/real 1761232332] req@00000000d07717c4 x1846780607051072/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232357 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232332.059661:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607051072/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232357 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232332.059669:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232332.059672:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232332.059675:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051072:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232332.059683:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051136:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232332.059705:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232332/real 1761232332] req@00000000b72feabb x1846780607051136/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232357 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232332.059717:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607051136/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232357 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232332.059726:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232332.059728:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232332.059730:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051136:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232332.059736:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051200:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232332.059757:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232332/real 1761232332] req@00000000659d78e3 x1846780607051200/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232357 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232332.059768:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607051200/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232357 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232332.059776:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232332.059778:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232332.059781:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051200:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232332.507095:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232333.531107:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232334.555125:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232335.579027:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232336.603185:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232337.179066:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232337.179079:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232337.179084:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232337.179098:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5209 00000100:00080000:2.0:1761232337.179102:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232337.179111:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232337.179160:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607051264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232337.179198:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232337.179202:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232337.179204:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232337.179209:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5209 00000100:00080000:2.0:1761232337.179211:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232337.179215:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232337.179225:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232337.179238:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607051328/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232337.179248:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232337.179251:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232337.179253:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232337.179257:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5209 00000100:00080000:2.0:1761232337.179259:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232337.179262:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232337.179266:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232337.179276:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607051392/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232337.179287:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232337.179292:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232337.179294:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232337.179323:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5209 00000100:00080000:2.0:1761232337.179326:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232337.179331:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232337.179337:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232337.179350:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607051264:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761232337.179354:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607051456/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232337.179466:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232337/real 1761232337] req@0000000094135f44 x1846780607051264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232362 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232337.179488:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607051264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232362 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232337.179501:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232337.179505:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232337.179510:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607051264:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232337.179517:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051328:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232337.179544:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232337/real 1761232337] req@0000000091106a97 x1846780607051328/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232362 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232337.179556:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607051328/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232362 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232337.179564:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232337.179567:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232337.179571:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051328:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232337.179581:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051392:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232337.179623:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232337/real 1761232337] req@0000000064978143 x1846780607051392/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232362 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232337.179645:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607051392/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232362 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232337.179661:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232337.179665:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232337.179671:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051392:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232337.179715:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051456:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232337.179767:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232337/real 1761232337] req@00000000c069b612 x1846780607051456/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232362 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232337.179781:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607051456/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232362 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232337.179792:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232337.179794:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232337.179797:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051456:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232337.627098:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232338.651009:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232339.675118:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232340.699192:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232341.723051:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232342.299016:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232342.299034:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232342.299039:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232342.299055:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5214 00000100:00080000:0.0:1761232342.299060:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232342.299071:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232342.299127:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607051520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232342.299164:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232342.299170:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232342.299172:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232342.299177:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5214 00000100:00080000:0.0:1761232342.299180:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232342.299183:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232342.299195:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232342.299208:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607051584/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232342.299218:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232342.299223:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232342.299226:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232342.299230:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5214 00000100:00080000:0.0:1761232342.299233:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232342.299237:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232342.299261:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232342.299276:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607051648/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232342.299288:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761232342.299290:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607051520:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761232342.299294:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232342.299297:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232342.299310:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5214 00000100:00080000:0.0:1761232342.299313:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232342.299317:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232342.299323:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232342.299344:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607051712/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232342.299404:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232342/real 1761232342] req@000000005a096598 x1846780607051520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232367 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232342.299418:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607051520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232367 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232342.299427:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232342.299429:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232342.299432:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607051520:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232342.299438:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051584:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232342.299453:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232342/real 1761232342] req@00000000275aeee5 x1846780607051584/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232367 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232342.299459:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607051584/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232367 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232342.299464:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232342.299466:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232342.299468:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051584:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232342.299488:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051648:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232342.299501:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232342/real 1761232342] req@000000002ef1f54f x1846780607051648/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232367 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232342.299508:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607051648/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232367 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232342.299513:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232342.299514:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232342.299516:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051648:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232342.299520:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051712:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232342.299530:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232342/real 1761232342] req@0000000023ab3a3f x1846780607051712/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232367 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232342.299536:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607051712/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232367 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232342.299541:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232342.299542:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232342.299544:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051712:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232342.747105:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232343.771185:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232344.795206:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232345.819047:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232346.843256:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232347.419077:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232347.419090:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232347.419094:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232347.419119:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5219 00000100:00080000:3.0:1761232347.419123:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232347.419132:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232347.419204:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607051776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232347.419257:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232347.419262:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232347.419264:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232347.419269:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5219 00000100:00080000:3.0:1761232347.419272:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232347.419276:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232347.419286:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232347.419300:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607051840/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232347.419310:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232347.419314:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232347.419315:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232347.419319:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5219 00000100:00080000:3.0:1761232347.419321:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232347.419324:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232347.419328:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232347.419337:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607051904/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232347.419345:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232347.419350:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232347.419352:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232347.419357:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5219 00000100:00080000:3.0:1761232347.419361:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232347.419365:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232347.419371:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232347.419381:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607051968/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232347.419402:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607051776:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232347.419528:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232347/real 1761232347] req@000000004e3f612a x1846780607051776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232372 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232347.419549:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607051776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232372 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232347.419561:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232347.419565:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232347.419569:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607051776:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232347.419577:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051840:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232347.419603:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232347/real 1761232347] req@0000000088473b3e x1846780607051840/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232372 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232347.419615:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607051840/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232372 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232347.419624:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232347.419626:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232347.419629:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051840:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232347.419636:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051904:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232347.419658:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232347/real 1761232347] req@000000003ad149cc x1846780607051904/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232372 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232347.419669:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607051904/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232372 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232347.419686:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232347.419688:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232347.419691:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051904:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232347.419724:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051968:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232347.419747:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232347/real 1761232347] req@00000000daa39bb1 x1846780607051968/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232372 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232347.419758:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607051968/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232372 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232347.419766:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232347.419769:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232347.419771:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607051968:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232347.867158:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232348.891043:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232349.915134:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232350.939099:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232351.963202:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232352.539041:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232352.539053:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232352.539057:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232352.539069:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5224 00000100:00080000:2.0:1761232352.539073:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232352.539081:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232352.539150:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607052032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232352.539196:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232352.539201:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232352.539203:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232352.539207:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5224 00000100:00080000:2.0:1761232352.539210:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232352.539213:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232352.539223:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232352.539237:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607052096/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232352.539248:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232352.539252:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232352.539254:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232352.539258:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5224 00000100:00080000:2.0:1761232352.539260:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232352.539264:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232352.539268:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232352.539278:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607052160/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232352.539289:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232352.539293:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232352.539295:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232352.539303:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607052032:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761232352.539303:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5224 00000100:00080000:2.0:1761232352.539307:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232352.539316:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232352.539322:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232352.539336:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607052224/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232352.539431:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232352/real 1761232352] req@0000000017386078 x1846780607052032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232377 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232352.539452:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607052032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232377 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232352.539465:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232352.539468:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232352.539472:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607052032:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232352.539480:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052096:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232352.539514:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232352/real 1761232352] req@00000000f38f27df x1846780607052096/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232377 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232352.539526:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607052096/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232377 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232352.539533:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232352.539536:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232352.539538:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052096:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232352.539566:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052160:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232352.539589:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232352/real 1761232352] req@00000000c069b612 x1846780607052160/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232377 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232352.539600:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607052160/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232377 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232352.539613:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232352.539617:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232352.539622:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052160:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232352.539632:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052224:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232352.539674:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232352/real 1761232352] req@0000000064978143 x1846780607052224/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232377 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232352.539695:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607052224/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232377 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232352.539710:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232352.539713:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232352.539718:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052224:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232352.987024:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232354.011163:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232355.035084:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232356.059172:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232357.083105:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232357.658995:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232357.659010:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232357.659014:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232357.659027:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5230 00000100:00080000:1.0:1761232357.659031:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232357.659039:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232357.659114:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607052288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232357.659152:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232357.659159:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232357.659162:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232357.659169:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5230 00000100:00080000:1.0:1761232357.659173:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232357.659179:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232357.659194:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232357.659218:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607052352/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232357.659236:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761232357.659241:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607052288:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761232357.659242:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232357.659244:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232357.659250:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5230 00000100:00080000:1.0:1761232357.659254:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232357.659259:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232357.659267:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232357.659281:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607052416/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232357.659295:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232357.659300:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232357.659302:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232357.659308:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5230 00000100:00080000:1.0:1761232357.659310:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232357.659315:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232357.659321:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232357.659336:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607052480/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232357.659355:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232357/real 1761232357] req@00000000659d78e3 x1846780607052288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232382 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232357.659383:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607052288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232382 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232357.659400:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232357.659405:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232357.659412:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607052288:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232357.659449:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052352:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232357.659493:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232357/real 1761232357] req@00000000b72feabb x1846780607052352/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232382 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232357.659506:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607052352/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232382 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232357.659516:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232357.659519:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232357.659522:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052352:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232357.659529:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052416:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232357.659552:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232357/real 1761232357] req@00000000d07717c4 x1846780607052416/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232382 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232357.659564:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607052416/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232382 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232357.659573:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232357.659575:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232357.659577:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052416:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232357.659583:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052480:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232357.659604:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232357/real 1761232357] req@00000000db10af3c x1846780607052480/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232382 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232357.659615:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607052480/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232382 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232357.659624:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232357.659626:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232357.659628:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052480:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232358.107154:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232359.131159:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232360.155090:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232361.179143:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232362.203251:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232362.778981:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232362.778996:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232362.779000:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232362.779049:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5235 00000100:00080000:1.0:1761232362.779053:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232362.779064:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232362.779113:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607052544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232362.779148:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232362.779153:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232362.779155:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232362.779160:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5235 00000100:00080000:1.0:1761232362.779162:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232362.779166:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232362.779176:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232362.779188:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607052608/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232362.779199:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232362.779210:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232362.779212:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232362.779216:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5235 00000100:00080000:1.0:1761232362.779219:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232362.779222:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232362.779226:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232362.779236:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607052672/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232362.779248:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232362.779252:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232362.779254:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232362.779260:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5235 00000100:00080000:1.0:1761232362.779262:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232362.779266:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232362.779271:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232362.779284:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607052544:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232362.779287:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607052736/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232362.779402:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232362/real 1761232362] req@00000000db10af3c x1846780607052544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232387 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232362.779431:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607052544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232387 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232362.779443:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232362.779447:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232362.779451:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607052544:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232362.779460:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052608:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232362.779487:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232362/real 1761232362] req@00000000d07717c4 x1846780607052608/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232387 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232362.779499:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607052608/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232387 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232362.779507:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232362.779510:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232362.779513:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052608:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232362.779518:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052672:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232362.779540:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232362/real 1761232362] req@00000000b72feabb x1846780607052672/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232387 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232362.779551:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607052672/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232387 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232362.779560:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232362.779562:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232362.779565:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052672:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232362.779597:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052736:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232362.779621:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232362/real 1761232362] req@00000000659d78e3 x1846780607052736/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232387 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232362.779633:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607052736/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232387 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232362.779641:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232362.779644:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232362.779646:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052736:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232363.227200:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232364.251167:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232365.275059:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232366.299059:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232367.323216:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232367.899070:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232367.899092:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232367.899097:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232367.899110:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5240 00000100:00080000:2.0:1761232367.899115:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232367.899125:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232367.899171:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607052800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232367.899206:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232367.899211:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232367.899213:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232367.899217:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5240 00000100:00080000:2.0:1761232367.899220:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232367.899224:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232367.899233:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232367.899246:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607052864/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232367.899257:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232367.899261:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232367.899263:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232367.899266:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5240 00000100:00080000:2.0:1761232367.899268:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232367.899271:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232367.899275:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232367.899286:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607052928/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232367.899296:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232367.899299:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232367.899301:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232367.899306:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5240 00000100:00080000:2.0:1761232367.899309:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232367.899314:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232367.899320:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232367.899328:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607052800:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761232367.899334:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607052992/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232367.899445:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232367/real 1761232367] req@0000000091106a97 x1846780607052800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232392 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232367.899466:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607052800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232392 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232367.899478:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232367.899482:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232367.899486:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607052800:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232367.899494:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052864:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232367.899520:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232367/real 1761232367] req@0000000094135f44 x1846780607052864/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232392 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232367.899533:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607052864/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232392 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232367.899549:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232367.899552:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232367.899555:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052864:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232367.899561:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052928:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232367.899582:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232367/real 1761232367] req@0000000064978143 x1846780607052928/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232392 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232367.899592:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607052928/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232392 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232367.899601:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232367.899603:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232367.899606:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052928:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232367.899635:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052992:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232367.899657:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232367/real 1761232367] req@00000000c069b612 x1846780607052992/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232392 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232367.899668:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607052992/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232392 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232367.899677:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232367.899679:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232367.899682:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607052992:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232368.347066:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232369.371089:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232370.395169:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232371.419117:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232372.443129:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232373.019031:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232373.019046:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232373.019050:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232373.019064:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5245 00000100:00080000:3.0:1761232373.019069:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232373.019078:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232373.019122:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607053056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232373.019159:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232373.019163:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232373.019165:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232373.019170:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5245 00000100:00080000:3.0:1761232373.019172:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232373.019176:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232373.019186:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232373.019198:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607053120/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232373.019208:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232373.019212:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232373.019214:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232373.019218:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5245 00000100:00080000:3.0:1761232373.019220:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232373.019223:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232373.019227:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232373.019236:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607053184/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232373.019246:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232373.019250:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232373.019253:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232373.019257:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5245 00000100:00080000:3.0:1761232373.019259:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232373.019262:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232373.019266:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232373.019276:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607053248/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232373.019296:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607053056:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232373.019411:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232373/real 1761232373] req@000000005a9bdefd x1846780607053056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232398 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232373.019432:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607053056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232398 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232373.019445:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232373.019449:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232373.019454:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607053056:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232373.019462:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053120:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232373.019489:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232373/real 1761232373] req@0000000018a8f5da x1846780607053120/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232398 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232373.019502:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607053120/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232398 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232373.019510:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232373.019512:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232373.019515:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053120:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232373.019522:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053184:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232373.019544:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232373/real 1761232373] req@00000000bd684ccc x1846780607053184/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232398 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232373.019556:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607053184/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232398 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232373.019565:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232373.019567:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232373.019570:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053184:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232373.019599:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053248:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232373.019623:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232373/real 1761232373] req@0000000086ccb527 x1846780607053248/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232398 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232373.019634:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607053248/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232398 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232373.019642:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232373.019644:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232373.019647:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053248:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232373.467194:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232374.491133:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232375.514997:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232376.539100:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232377.563143:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232378.139097:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232378.139110:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232378.139115:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232378.139127:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5250 00000100:00080000:2.0:1761232378.139132:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232378.139140:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232378.139184:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607053312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232378.139225:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232378.139229:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232378.139231:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232378.139236:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5250 00000100:00080000:2.0:1761232378.139239:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232378.139243:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232378.139253:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232378.139269:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607053376/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232378.139279:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232378.139284:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232378.139286:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232378.139291:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5250 00000100:00080000:2.0:1761232378.139293:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232378.139298:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232378.139303:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232378.139314:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607053440/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232378.139323:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232378.139327:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232378.139328:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232378.139332:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5250 00000100:00080000:2.0:1761232378.139335:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232378.139338:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232378.139345:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232378.139353:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607053312:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761232378.139359:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607053504/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232378.139489:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232378/real 1761232378] req@00000000f38f27df x1846780607053312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232403 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232378.139512:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607053312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232403 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232378.139526:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232378.139530:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232378.139535:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607053312:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232378.139544:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053376:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232378.139576:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232378/real 1761232378] req@0000000017386078 x1846780607053376/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232403 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232378.139589:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607053376/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232403 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232378.139597:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232378.139599:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232378.139602:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053376:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232378.139608:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053440:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232378.139630:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232378/real 1761232378] req@00000000c069b612 x1846780607053440/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232403 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232378.139647:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607053440/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232403 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232378.139662:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232378.139667:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232378.139672:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053440:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232378.139715:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053504:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232378.139764:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232378/real 1761232378] req@0000000064978143 x1846780607053504/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232403 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232378.139779:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607053504/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232403 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232378.139789:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232378.139791:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232378.139795:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053504:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232378.587126:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232379.611173:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232380.635207:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232381.659156:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232382.683178:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232383.259023:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232383.259038:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232383.259043:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232383.259057:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5255 00000100:00080000:3.0:1761232383.259061:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232383.259070:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232383.259114:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607053568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232383.259155:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232383.259160:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232383.259162:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232383.259166:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5255 00000100:00080000:3.0:1761232383.259169:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232383.259173:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232383.259186:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232383.259199:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607053632/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232383.259209:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232383.259213:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232383.259215:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232383.259218:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5255 00000100:00080000:3.0:1761232383.259220:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232383.259223:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232383.259227:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232383.259238:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607053696/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232383.259247:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232383.259251:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232383.259253:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232383.259257:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5255 00000100:00080000:3.0:1761232383.259260:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232383.259265:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232383.259269:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232383.259279:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607053760/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232383.259305:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607053568:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232383.259416:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232383/real 1761232383] req@00000000c7e4e432 x1846780607053568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232408 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232383.259437:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607053568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232408 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232383.259450:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232383.259453:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232383.259458:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607053568:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232383.259466:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053632:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232383.259493:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232383/real 1761232383] req@00000000f87d0618 x1846780607053632/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232408 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232383.259505:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607053632/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232408 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232383.259513:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232383.259517:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232383.259520:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053632:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232383.259526:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053696:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232383.259547:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232383/real 1761232383] req@000000007361665d x1846780607053696/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232408 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232383.259558:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607053696/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232408 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232383.259566:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232383.259568:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232383.259571:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053696:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232383.259602:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053760:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232383.259624:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232383/real 1761232383] req@0000000031e043f4 x1846780607053760/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232408 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232383.259636:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607053760/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232408 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232383.259645:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232383.259647:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232383.259650:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053760:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232383.707082:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232384.731103:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232385.755013:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232386.779089:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232387.803135:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232388.378968:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232388.378982:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232388.378987:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232388.379001:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5260 00000100:00080000:3.0:1761232388.379006:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232388.379015:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232388.379061:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607053824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232388.379098:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232388.379103:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232388.379105:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232388.379109:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5260 00000100:00080000:3.0:1761232388.379112:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232388.379115:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232388.379125:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232388.379137:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607053888/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232388.379147:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232388.379150:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232388.379152:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232388.379156:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5260 00000100:00080000:3.0:1761232388.379167:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232388.379171:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232388.379175:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232388.379187:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607053952/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232388.379197:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232388.379202:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232388.379204:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232388.379208:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5260 00000100:00080000:3.0:1761232388.379211:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232388.379216:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232388.379220:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232388.379231:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607054016/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232388.379259:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607053824:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232388.379373:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053888:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232388.379403:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053952:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232388.379436:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054016:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232388.827075:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232389.851213:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232390.875091:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761232391.451358:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232388/real 1761232391] req@0000000083f16a6b x1846780607054016/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232413 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232391.451392:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607054016/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232413 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232391.451414:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232391.451421:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232391.451428:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054016:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232391.451439:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232388/real 1761232391] req@00000000a58ca11e x1846780607053824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232413 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232391.451462:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607053824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232413 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232391.451475:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232391.451479:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232391.451484:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607053824:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232391.451494:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232388/real 1761232391] req@000000001f8e688e x1846780607053888/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232413 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232391.451514:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607053888/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232413 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232391.451528:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232391.451533:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232391.451538:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053888:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232391.451549:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232388/real 1761232391] req@0000000090b40c66 x1846780607053952/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232413 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232391.451570:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607053952/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232413 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232391.451597:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232391.451601:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232391.451606:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607053952:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232391.899131:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232392.923147:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232393.498977:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232393.498991:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232393.498997:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232393.499012:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5265 00000100:00080000:3.0:1761232393.499017:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232393.499027:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232393.499070:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607054080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232393.499106:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232393.499111:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232393.499113:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232393.499118:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5265 00000100:00080000:3.0:1761232393.499120:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232393.499123:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232393.499133:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232393.499147:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607054144/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232393.499162:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232393.499166:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232393.499169:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232393.499172:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607054080:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:3.0:1761232393.499178:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5265 00000100:00080000:3.0:1761232393.499186:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232393.499189:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232393.499196:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232393.499215:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607054208/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232393.499231:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232393.499239:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232393.499241:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232393.499246:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5265 00000100:00080000:3.0:1761232393.499251:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232393.499257:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232393.499264:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232393.499292:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232393/real 1761232393] req@000000006572b591 x1846780607054080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232418 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232393.499318:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607054272/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232393.499322:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607054080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232418 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232393.499377:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232393.499386:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232393.499395:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607054080:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232393.499440:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054144:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232393.499491:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232393/real 1761232393] req@00000000fc01c41c x1846780607054144/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232418 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232393.499512:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607054144/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232418 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232393.499528:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232393.499532:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232393.499538:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054144:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232393.499548:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054208:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232393.499593:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232393/real 1761232393] req@00000000408f088c x1846780607054208/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232418 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232393.499615:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607054208/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232418 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232393.499631:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232393.499635:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232393.499640:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054208:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232393.499651:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054272:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232393.499723:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232393/real 1761232393] req@000000002ce934d3 x1846780607054272/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232418 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232393.499736:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607054272/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232418 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232393.499745:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232393.499747:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232393.499750:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054272:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232393.947102:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232394.971153:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232395.995130:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232397.019148:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232398.043130:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232398.618903:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232398.618920:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232398.618927:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232398.618944:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5271 00000100:00080000:3.0:1761232398.618949:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232398.618962:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232398.619024:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607054336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232398.619065:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232398.619070:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232398.619072:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232398.619078:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5271 00000100:00080000:3.0:1761232398.619081:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232398.619084:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232398.619098:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232398.619111:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607054400/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232398.619122:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232398.619126:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232398.619127:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232398.619131:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5271 00000100:00080000:3.0:1761232398.619133:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232398.619137:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232398.619141:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232398.619154:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607054464/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232398.619164:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232398.619167:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232398.619169:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232398.619178:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5271 00000100:00080000:3.0:1761232398.619181:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232398.619187:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232398.619193:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232398.619206:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607054528/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232398.619214:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607054336:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232398.619339:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232398/real 1761232398] req@000000002ce934d3 x1846780607054336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232423 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232398.619362:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607054336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232423 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232398.619375:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232398.619380:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232398.619384:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607054336:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232398.619394:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054400:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232398.619421:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232398/real 1761232398] req@00000000408f088c x1846780607054400/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232423 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232398.619434:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607054400/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232423 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232398.619442:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232398.619444:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232398.619447:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054400:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232398.619453:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054464:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232398.619475:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232398/real 1761232398] req@00000000fc01c41c x1846780607054464/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232423 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232398.619487:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607054464/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232423 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232398.619496:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232398.619499:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232398.619502:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054464:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232398.619535:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054528:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232398.619559:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232398/real 1761232398] req@000000006572b591 x1846780607054528/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232423 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232398.619571:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607054528/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232423 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232398.619579:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232398.619582:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232398.619584:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054528:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232399.067074:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232400.091068:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232401.115170:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232402.139072:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232403.163152:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232403.739027:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232403.739036:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232403.739040:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232403.739049:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5276 00000100:00080000:2.0:1761232403.739052:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232403.739058:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232403.739093:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607054592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232403.739122:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232403.739125:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232403.739126:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232403.739129:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5276 00000100:00080000:2.0:1761232403.739131:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232403.739133:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232403.739140:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232403.739149:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607054656/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232403.739157:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232403.739160:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232403.739161:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232403.739163:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5276 00000100:00080000:2.0:1761232403.739165:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232403.739168:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232403.739172:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232403.739184:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607054720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232403.739190:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607054592:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:2.0:1761232403.739196:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232403.739202:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232403.739203:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232403.739206:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5276 00000100:00080000:2.0:1761232403.739208:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232403.739211:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232403.739215:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232403.739228:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607054784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232403.739311:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232403/real 1761232403] req@0000000094135f44 x1846780607054592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232428 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232403.739325:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607054592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232428 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232403.739334:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232403.739337:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232403.739340:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607054592:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232403.739345:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054656:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232403.739361:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232403/real 1761232403] req@0000000091106a97 x1846780607054656/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232428 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232403.739374:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607054656/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232428 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232403.739378:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232403.739380:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232403.739381:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054656:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232403.739405:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054720:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232403.739418:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232403/real 1761232403] req@0000000064978143 x1846780607054720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232428 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232403.739425:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607054720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232428 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232403.739431:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232403.739432:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232403.739433:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054720:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232403.739437:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054784:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232403.739448:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232403/real 1761232403] req@00000000c069b612 x1846780607054784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232428 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232403.739454:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607054784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232428 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232403.739459:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232403.739460:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232403.739463:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054784:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232404.187087:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232405.210979:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232406.235129:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232407.259070:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232408.283040:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232408.858990:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232408.859001:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232408.859004:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232408.859013:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5281 00000100:00080000:0.0:1761232408.859018:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232408.859024:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232408.859056:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607054848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232408.859080:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232408.859083:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232408.859084:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232408.859088:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5281 00000100:00080000:0.0:1761232408.859089:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232408.859091:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232408.859099:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232408.859117:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607054912/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232408.859123:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232408.859126:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232408.859127:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232408.859129:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5281 00000100:00080000:0.0:1761232408.859130:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232408.859132:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232408.859135:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232408.859140:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607054976/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232408.859145:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232408.859148:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232408.859150:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232408.859153:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5281 00000100:00080000:0.0:1761232408.859156:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232408.859158:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232408.859161:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232408.859169:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607055040/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232408.859180:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607054848:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232408.859261:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232408/real 1761232408] req@00000000271a9d8e x1846780607054848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232433 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232408.859274:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607054848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232433 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232408.859282:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232408.859284:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232408.859287:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607054848:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232408.859292:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054912:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232408.859307:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232408/real 1761232408] req@000000000243251f x1846780607054912/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232433 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232408.859315:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607054912/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232433 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232408.859319:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232408.859321:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232408.859322:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054912:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232408.859326:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054976:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232408.859338:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232408/real 1761232408] req@0000000099514377 x1846780607054976/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232433 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232408.859344:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607054976/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232433 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232408.859351:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232408.859353:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232408.859355:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607054976:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232408.859375:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055040:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232408.859388:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232408/real 1761232408] req@0000000001b2f813 x1846780607055040/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232433 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232408.859394:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607055040/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232433 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232408.859400:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232408.859401:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232408.859402:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055040:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232409.307124:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232410.330972:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232411.355088:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232412.378984:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232413.403009:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232413.978912:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232413.978924:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232413.978928:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232413.978937:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5286 00000100:00080000:0.0:1761232413.978941:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232413.978948:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232413.978982:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607055104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232413.979006:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232413.979010:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232413.979012:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232413.979015:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5286 00000100:00080000:0.0:1761232413.979018:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232413.979021:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232413.979030:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232413.979040:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607055168/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232413.979049:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232413.979052:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232413.979053:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232413.979057:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5286 00000100:00080000:0.0:1761232413.979059:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232413.979063:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232413.979067:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232413.979078:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607055232/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232413.979085:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607055104:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761232413.979090:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232413.979094:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232413.979095:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232413.979099:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5286 00000100:00080000:0.0:1761232413.979102:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232413.979105:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232413.979109:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232413.979120:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607055296/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232413.979161:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232413/real 1761232413] req@00000000b8f2be78 x1846780607055104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232438 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232413.979173:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607055104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232438 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232413.979180:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232413.979182:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232413.979185:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607055104:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232413.979189:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055168:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232413.979205:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232413/real 1761232413] req@00000000fcce404c x1846780607055168/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232438 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232413.979212:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607055168/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232438 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232413.979216:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232413.979217:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232413.979219:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055168:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232413.979236:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055232:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232413.979249:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232413/real 1761232413] req@000000007f60e210 x1846780607055232/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232438 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232413.979255:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607055232/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232438 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232413.979260:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232413.979261:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232413.979262:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055232:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232413.979265:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055296:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232413.979275:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232413/real 1761232413] req@0000000001b2f813 x1846780607055296/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232438 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232413.979282:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607055296/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232438 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232413.979286:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232413.979288:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232413.979289:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055296:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232414.427016:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232415.451018:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232416.475009:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232417.498989:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232418.522973:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232419.098960:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232419.098969:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232419.098972:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232419.098980:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5291 00000100:00080000:3.0:1761232419.098983:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232419.098989:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232419.099019:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607055360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232419.099041:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232419.099044:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232419.099045:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232419.099048:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5291 00000100:00080000:3.0:1761232419.099050:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232419.099052:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232419.099058:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232419.099066:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607055424/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232419.099072:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232419.099075:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232419.099076:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232419.099079:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5291 00000100:00080000:3.0:1761232419.099080:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232419.099082:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232419.099086:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232419.099089:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607055360:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232419.099096:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607055488/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232419.099103:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232419.099104:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232419.099105:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232419.099107:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5291 00000100:00080000:3.0:1761232419.099108:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232419.099110:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232419.099113:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232419.099120:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607055552/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232419.099166:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232419/real 1761232419] req@0000000090b40c66 x1846780607055360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232444 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232419.099178:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607055360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232444 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232419.099185:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232419.099187:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232419.099190:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607055360:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232419.099194:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055424:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232419.099209:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232419/real 1761232419] req@000000001f8e688e x1846780607055424/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232444 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232419.099215:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607055424/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232444 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232419.099219:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232419.099221:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232419.099222:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055424:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232419.099239:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055488:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232419.099251:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232419/real 1761232419] req@00000000a58ca11e x1846780607055488/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232444 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232419.099262:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607055488/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232444 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232419.099267:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232419.099268:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232419.099270:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055488:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232419.099273:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055552:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232419.099283:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232419/real 1761232419] req@0000000083f16a6b x1846780607055552/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232444 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232419.099289:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607055552/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232444 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232419.099294:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232419.099295:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232419.099297:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055552:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232419.547032:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232420.571015:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232421.595106:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232422.619021:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232423.643000:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232424.218886:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232424.218895:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232424.218899:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232424.218907:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5296 00000100:00080000:3.0:1761232424.218910:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232424.218915:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232424.218945:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607055616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232424.218965:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232424.218968:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232424.218969:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232424.218972:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5296 00000100:00080000:3.0:1761232424.218973:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232424.218975:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232424.218982:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232424.218989:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607055680/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232424.218994:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232424.218996:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232424.218998:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232424.219000:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5296 00000100:00080000:3.0:1761232424.219001:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232424.219003:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232424.219005:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232424.219013:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607055744/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232424.219018:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232424.219020:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232424.219021:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232424.219023:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5296 00000100:00080000:3.0:1761232424.219024:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232424.219028:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232424.219032:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232424.219033:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607055616:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232424.219040:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607055808/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232424.219117:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232424/real 1761232424] req@0000000031e043f4 x1846780607055616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232449 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232424.219136:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607055616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232449 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232424.219145:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232424.219148:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232424.219152:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607055616:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232424.219159:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055680:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232424.219185:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232424/real 1761232424] req@000000007361665d x1846780607055680/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232449 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232424.219196:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607055680/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232449 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232424.219205:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232424.219207:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232424.219210:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055680:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232424.219216:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055744:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232424.219240:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232424/real 1761232424] req@00000000f87d0618 x1846780607055744/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232449 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232424.219251:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607055744/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232449 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232424.219260:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232424.219263:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232424.219266:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055744:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232424.219290:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055808:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232424.219329:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232424/real 1761232424] req@00000000c7e4e432 x1846780607055808/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232449 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232424.219341:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607055808/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232449 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232424.219350:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232424.219352:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232424.219355:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055808:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232424.667040:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232425.691034:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232426.714976:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232427.739054:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232428.763077:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232429.339001:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232429.339013:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232429.339017:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232429.339028:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5301 00000100:00080000:1.0:1761232429.339032:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232429.339039:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232429.339085:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607055872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232429.339116:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232429.339119:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232429.339120:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232429.339123:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5301 00000100:00080000:1.0:1761232429.339125:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232429.339128:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232429.339137:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232429.339147:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607055936/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232429.339155:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232429.339158:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232429.339159:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232429.339166:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5301 00000100:00100000:3.0:1761232429.339166:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607055872:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761232429.339173:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232429.339181:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232429.339185:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232429.339194:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607056000/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232429.339200:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232429.339203:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232429.339204:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232429.339207:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5301 00000100:00080000:1.0:1761232429.339208:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232429.339210:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232429.339212:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232429.339221:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607056064/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232429.339278:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232429/real 1761232429] req@00000000659d78e3 x1846780607055872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232454 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232429.339293:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607055872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232454 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232429.339301:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232429.339303:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232429.339306:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607055872:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232429.339326:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055936:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232429.339345:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232429/real 1761232429] req@00000000b72feabb x1846780607055936/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232454 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232429.339352:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607055936/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232454 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232429.339356:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232429.339357:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232429.339359:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607055936:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232429.339362:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056000:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232429.339377:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232429/real 1761232429] req@00000000d07717c4 x1846780607056000/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232454 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232429.339383:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607056000/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232454 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232429.339388:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232429.339389:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232429.339391:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056000:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232429.339393:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056064:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232429.339406:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232429/real 1761232429] req@00000000db10af3c x1846780607056064/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232454 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232429.339412:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607056064/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232454 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232429.339416:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232429.339417:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232429.339419:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056064:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232429.787079:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232430.811284:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232431.835151:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232432.859086:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232433.883069:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232434.458903:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232434.458914:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232434.458918:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232434.458927:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5306 00000100:00080000:1.0:1761232434.458931:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232434.458937:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232434.458971:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607056128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232434.458994:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232434.458999:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232434.459000:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232434.459004:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5306 00000100:00080000:1.0:1761232434.459006:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232434.459010:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232434.459018:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232434.459028:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607056192/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232434.459037:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232434.459040:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232434.459042:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232434.459045:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5306 00000100:00080000:1.0:1761232434.459047:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232434.459051:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232434.459055:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232434.459066:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607056256/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232434.459075:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607056128:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761232434.459077:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232434.459083:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232434.459084:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232434.459089:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5306 00000100:00080000:1.0:1761232434.459091:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232434.459094:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232434.459098:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232434.459110:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607056320/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232434.459150:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232434/real 1761232434] req@00000000db10af3c x1846780607056128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232459 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232434.459162:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607056128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232459 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232434.459170:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232434.459172:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232434.459175:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607056128:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232434.459180:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056192:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232434.459211:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232434/real 1761232434] req@00000000d07717c4 x1846780607056192/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232459 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232434.459218:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607056192/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232459 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232434.459223:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232434.459224:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232434.459226:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056192:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232434.459243:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056256:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232434.459258:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232434/real 1761232434] req@00000000b72feabb x1846780607056256/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232459 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232434.459263:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607056256/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232459 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232434.459268:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232434.459270:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232434.459271:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056256:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232434.459274:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056320:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232434.459284:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232434/real 1761232434] req@00000000659d78e3 x1846780607056320/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232459 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232434.459294:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607056320/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232459 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232434.459300:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232434.459301:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232434.459302:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056320:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232434.907010:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232435.931109:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232436.955210:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232437.979113:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232439.003070:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232439.578928:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232439.578937:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232439.578940:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232439.578948:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5311 00000100:00080000:1.0:1761232439.578951:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232439.578956:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232439.578990:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607056384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232439.579010:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232439.579013:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232439.579014:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232439.579017:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5311 00000100:00080000:1.0:1761232439.579018:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232439.579020:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232439.579026:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232439.579033:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607056448/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232439.579038:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232439.579041:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232439.579042:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232439.579045:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5311 00000100:00080000:1.0:1761232439.579046:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232439.579048:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232439.579052:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232439.579062:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607056512/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232439.579063:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607056384:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761232439.579075:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232439.579077:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232439.579078:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232439.579081:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5311 00000100:00080000:1.0:1761232439.579082:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232439.579084:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232439.579087:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232439.579093:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607056576/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232439.579167:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232439/real 1761232439] req@00000000659d78e3 x1846780607056384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232464 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232439.579181:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607056384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232464 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232439.579188:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232439.579190:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232439.579192:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607056384:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232439.579197:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056448:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232439.579214:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232439/real 1761232439] req@00000000b72feabb x1846780607056448/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232464 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232439.579220:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607056448/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232464 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232439.579225:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232439.579226:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232439.579228:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056448:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232439.579246:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056512:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232439.579259:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232439/real 1761232439] req@00000000d07717c4 x1846780607056512/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232464 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232439.579265:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607056512/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232464 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232439.579271:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232439.579272:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232439.579273:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056512:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232439.579276:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056576:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232439.579286:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232439/real 1761232439] req@00000000db10af3c x1846780607056576/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232464 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232439.579292:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607056576/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232464 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232439.579297:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232439.579298:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232439.579299:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056576:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232440.027183:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232441.051233:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232442.075070:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232443.099190:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232444.123152:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232444.699153:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232444.699166:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232444.699172:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232444.699184:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5317 00000100:00080000:2.0:1761232444.699188:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232444.699198:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232444.699246:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607056640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232444.699284:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232444.699288:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232444.699290:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232444.699295:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5317 00000100:00080000:2.0:1761232444.699298:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232444.699302:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232444.699311:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232444.699324:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607056704/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232444.699334:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232444.699338:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232444.699340:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232444.699345:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5317 00000100:00080000:2.0:1761232444.699347:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232444.699351:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232444.699356:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232444.699369:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607056768/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232444.699379:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232444.699383:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232444.699385:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232444.699390:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5317 00000100:00080000:2.0:1761232444.699393:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232444.699396:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232444.699402:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232444.699406:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607056640:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761232444.699417:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607056832/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232444.699522:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232444/real 1761232444] req@0000000017386078 x1846780607056640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232469 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232444.699543:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607056640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232469 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232444.699557:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232444.699561:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232444.699565:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607056640:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232444.699573:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056704:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232444.699599:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232444/real 1761232444] req@00000000f38f27df x1846780607056704/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232469 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232444.699612:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607056704/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232469 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232444.699620:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232444.699622:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232444.699625:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056704:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232444.699630:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056768:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232444.699651:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232444/real 1761232444] req@00000000c069b612 x1846780607056768/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232469 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232444.699663:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607056768/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232469 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232444.699672:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232444.699674:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232444.699677:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056768:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232444.699707:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056832:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232444.699730:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232444/real 1761232444] req@0000000064978143 x1846780607056832/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232469 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232444.699742:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607056832/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232469 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232444.699751:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232444.699753:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232444.699756:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056832:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232445.147160:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232446.171155:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232447.195158:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232448.219099:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232449.243170:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232449.819124:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232449.819136:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232449.819141:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232449.819154:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5322 00000100:00080000:3.0:1761232449.819158:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232449.819167:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232449.819212:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607056896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232449.819251:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232449.819256:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232449.819258:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232449.819263:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5322 00000100:00080000:3.0:1761232449.819265:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232449.819268:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232449.819278:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232449.819290:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607056960/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232449.819301:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232449.819305:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232449.819307:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232449.819310:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5322 00000100:00080000:3.0:1761232449.819312:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232449.819315:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232449.819320:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232449.819328:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607057024/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232449.819337:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232449.819340:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232449.819342:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232449.819346:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5322 00000100:00080000:3.0:1761232449.819349:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232449.819353:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232449.819358:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232449.819368:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607057088/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232449.819396:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607056896:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232449.819509:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232449/real 1761232449] req@0000000086ccb527 x1846780607056896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232474 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232449.819530:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607056896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232474 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232449.819543:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232449.819547:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232449.819552:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607056896:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232449.819560:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056960:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232449.819587:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232449/real 1761232449] req@00000000bd684ccc x1846780607056960/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232474 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232449.819609:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607056960/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232474 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232449.819617:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232449.819620:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232449.819623:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607056960:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232449.819629:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057024:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232449.819653:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232449/real 1761232449] req@0000000018a8f5da x1846780607057024/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232474 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232449.819665:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607057024/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232474 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232449.819674:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232449.819676:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232449.819679:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057024:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232449.819709:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057088:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232449.819734:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232449/real 1761232449] req@000000005a9bdefd x1846780607057088/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232474 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232449.819745:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607057088/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232474 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232449.819753:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232449.819755:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232449.819759:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057088:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232450.267276:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232451.291174:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232452.315167:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232453.339152:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232454.363130:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232454.938976:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232454.938985:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232454.938989:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232454.938997:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5327 00000100:00080000:1.0:1761232454.939000:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232454.939006:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232454.939040:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607057152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232454.939065:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232454.939068:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232454.939069:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232454.939072:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5327 00000100:00080000:1.0:1761232454.939074:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232454.939075:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232454.939082:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232454.939091:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607057216/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232454.939098:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232454.939100:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232454.939101:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232454.939104:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5327 00000100:00080000:1.0:1761232454.939106:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232454.939109:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1761232454.939115:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607057152:192.168.201.104@tcp:250:kworker/u8:2.0 00000008:00100000:1.0:1761232454.939116:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232454.939127:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607057280/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232454.939133:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232454.939135:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232454.939137:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232454.939140:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5327 00000100:00080000:1.0:1761232454.939141:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232454.939143:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232454.939148:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232454.939156:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607057344/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232454.939288:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057216:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232454.939321:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057280:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232454.939344:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057344:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232455.387216:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232456.411210:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232457.435146:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761232458.011463:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232454/real 1761232458] req@00000000d07717c4 x1846780607057216/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232479 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232458.011522:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607057216/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232479 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232458.011542:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232458.011549:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232458.011556:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057216:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232458.011568:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232454/real 1761232458] req@00000000b72feabb x1846780607057280/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232479 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232458.011591:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607057280/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232479 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232458.011606:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232458.011611:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232458.011616:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057280:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232458.011626:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232454/real 1761232458] req@00000000659d78e3 x1846780607057344/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232479 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232458.011658:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607057344/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232479 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232458.011673:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232458.011677:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232458.011683:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057344:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232458.011698:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232454/real 1761232458] req@00000000db10af3c x1846780607057152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232479 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232458.011717:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607057152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232479 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232458.011731:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232458.011735:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232458.011740:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607057152:192.168.201.104@tcp:250:kworker/u8:2.0 00010000:00010000:0.0:1761232458.459049:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232459.483151:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232460.059044:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232460.059053:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232460.059057:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232460.059066:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5332 00000100:00080000:3.0:1761232460.059069:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232460.059075:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232460.059106:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607057408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232460.059137:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232460.059141:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232460.059142:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232460.059145:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5332 00000100:00080000:3.0:1761232460.059147:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232460.059149:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232460.059156:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232460.059166:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607057472/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232460.059174:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232460.059177:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232460.059178:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232460.059183:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5332 00000100:00080000:3.0:1761232460.059185:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232460.059187:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232460.059190:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232460.059197:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607057536/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232460.059206:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232460.059207:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232460.059208:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232460.059211:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5332 00000100:00080000:3.0:1761232460.059213:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232460.059215:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232460.059218:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232460.059232:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607057408:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232460.059232:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607057600/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232460.059421:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232460/real 1761232460] req@00000000daa39bb1 x1846780607057408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232485 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232460.059455:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607057408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232485 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232460.059476:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232460.059483:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232460.059491:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607057408:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232460.059505:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057472:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232460.059566:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232460/real 1761232460] req@000000003ad149cc x1846780607057472/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232485 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232460.059600:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607057472/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232485 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232460.059616:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232460.059621:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232460.059627:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057472:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232460.059700:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057536:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232460.059754:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232460/real 1761232460] req@0000000088473b3e x1846780607057536/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232485 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232460.059773:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607057536/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232485 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232460.059787:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232460.059791:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232460.059795:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057536:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232460.059805:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057600:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232460.059896:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232460/real 1761232460] req@000000004e3f612a x1846780607057600/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232485 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232460.059952:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607057600/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232485 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232460.059966:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232460.059970:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232460.059976:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057600:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232460.507006:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232461.530972:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232462.555015:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232463.579108:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232464.603144:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232465.179009:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232465.179027:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232465.179031:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232465.179042:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5337 00000100:00080000:0.0:1761232465.179046:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232465.179053:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232465.179092:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607057664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232465.179120:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232465.179124:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232465.179125:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232465.179129:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5337 00000100:00080000:0.0:1761232465.179131:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232465.179134:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232465.179143:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232465.179175:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607057728/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232465.179187:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232465.179190:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232465.179192:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232465.179196:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5337 00000100:00080000:0.0:1761232465.179200:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232465.179204:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232465.179211:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232465.179224:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607057792/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232465.179233:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232465.179237:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232465.179238:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232465.179244:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5337 00000100:00080000:0.0:1761232465.179246:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232465.179250:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232465.179254:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232465.179274:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607057856/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232465.179275:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607057664:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232465.179430:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232465/real 1761232465] req@0000000099514377 x1846780607057664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232490 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232465.179453:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607057664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232490 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232465.179466:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232465.179470:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232465.179474:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607057664:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232465.179482:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057728:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232465.179504:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232465/real 1761232465] req@000000000243251f x1846780607057728/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232490 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232465.179513:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607057728/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232490 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232465.179520:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232465.179523:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232465.179525:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057728:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232465.179530:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057792:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232465.179547:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232465/real 1761232465] req@00000000271a9d8e x1846780607057792/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232490 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232465.179555:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607057792/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232490 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232465.179564:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232465.179566:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232465.179569:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057792:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232465.179603:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057856:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232465.179621:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232465/real 1761232465] req@0000000023ab3a3f x1846780607057856/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232490 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232465.179629:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607057856/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232490 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232465.179637:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232465.179639:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232465.179641:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057856:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232465.627093:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232466.651058:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232467.675080:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232468.699159:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232469.723092:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232470.299106:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232470.299118:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232470.299123:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232470.299136:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5342 00000100:00080000:3.0:1761232470.299140:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232470.299150:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232470.299199:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607057920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232470.299239:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232470.299244:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232470.299246:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232470.299259:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5342 00000100:00080000:3.0:1761232470.299262:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232470.299266:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232470.299277:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232470.299293:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607057984/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232470.299306:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232470.299310:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232470.299312:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232470.299317:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5342 00000100:00080000:3.0:1761232470.299319:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1761232470.299327:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607057920:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:3.0:1761232470.299328:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232470.299340:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232470.299356:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607058048/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232470.299368:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232470.299372:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232470.299374:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232470.299379:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5342 00000100:00080000:3.0:1761232470.299382:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232470.299386:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232470.299392:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232470.299403:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607058112/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232470.299451:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232470/real 1761232470] req@000000004e3f612a x1846780607057920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232495 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232470.299472:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607057920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232495 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232470.299484:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232470.299488:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232470.299492:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607057920:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232470.299501:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057984:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232470.299532:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232470/real 1761232470] req@0000000088473b3e x1846780607057984/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232495 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232470.299544:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607057984/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232495 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232470.299552:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232470.299555:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232470.299558:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607057984:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232470.299589:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058048:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232470.299613:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232470/real 1761232470] req@000000003ad149cc x1846780607058048/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232495 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232470.299625:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607058048/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232495 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232470.299634:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232470.299636:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232470.299639:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058048:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232470.299645:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058112:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232470.299665:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232470/real 1761232470] req@00000000daa39bb1 x1846780607058112/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232495 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232470.299677:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607058112/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232495 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232470.299686:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232470.299689:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232470.299691:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058112:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232470.747095:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232471.771132:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232472.795036:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232473.819012:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232474.843063:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232475.418968:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232475.418981:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232475.418986:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232475.418999:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5347 00000100:00080000:3.0:1761232475.419004:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232475.419012:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232475.419057:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607058176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232475.419091:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232475.419096:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232475.419098:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232475.419103:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5347 00000100:00080000:3.0:1761232475.419106:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232475.419109:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232475.419119:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232475.419131:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607058240/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232475.419141:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232475.419146:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232475.419148:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232475.419152:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5347 00000100:00080000:3.0:1761232475.419154:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232475.419157:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232475.419161:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232475.419173:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607058304/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232475.419183:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232475.419187:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232475.419189:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232475.419193:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5347 00000100:00080000:3.0:1761232475.419196:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232475.419200:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232475.419205:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232475.419215:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607058368/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232475.419230:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607058176:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232475.419342:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232475/real 1761232475] req@000000005a9bdefd x1846780607058176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232500 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232475.419363:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607058176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232500 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232475.419376:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232475.419379:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232475.419385:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607058176:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232475.419392:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058240:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232475.419418:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232475/real 1761232475] req@0000000018a8f5da x1846780607058240/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232500 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232475.419431:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607058240/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232500 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232475.419440:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232475.419442:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232475.419445:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058240:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232475.419451:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058304:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232475.419472:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232475/real 1761232475] req@00000000bd684ccc x1846780607058304/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232500 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232475.419492:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607058304/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232500 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232475.419501:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232475.419504:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232475.419506:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058304:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232475.419539:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058368:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232475.419561:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232475/real 1761232475] req@0000000086ccb527 x1846780607058368/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232500 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232475.419572:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607058368/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232500 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232475.419581:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232475.419583:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232475.419586:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058368:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232475.867171:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232476.891159:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232477.915067:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232478.939151:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232479.963165:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232480.539047:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232480.539059:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232480.539062:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232480.539072:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5352 00000100:00080000:0.0:1761232480.539075:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232480.539082:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232480.539116:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607058432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232480.539157:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232480.539160:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232480.539161:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232480.539164:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5352 00000100:00080000:0.0:1761232480.539165:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232480.539167:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232480.539173:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232480.539180:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607058496/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232480.539187:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232480.539189:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232480.539190:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232480.539192:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5352 00000100:00080000:0.0:1761232480.539193:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232480.539195:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232480.539197:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232480.539204:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607058560/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232480.539208:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232480.539209:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232480.539210:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232480.539212:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5352 00000100:00080000:0.0:1761232480.539214:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232480.539216:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232480.539218:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232480.539224:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607058624/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232480.539243:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607058432:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232480.539315:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232480/real 1761232480] req@000000002ef1f54f x1846780607058432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232505 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232480.539327:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607058432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232505 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232480.539334:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232480.539336:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232480.539339:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607058432:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232480.539344:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058496:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232480.539357:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232480/real 1761232480] req@00000000275aeee5 x1846780607058496/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232505 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232480.539364:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607058496/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232505 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232480.539369:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232480.539370:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232480.539371:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058496:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232480.539374:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058560:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232480.539385:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232480/real 1761232480] req@000000005a096598 x1846780607058560/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232505 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232480.539390:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607058560/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232505 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232480.539395:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232480.539397:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232480.539398:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058560:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232480.539401:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058624:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232480.539411:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232480/real 1761232480] req@0000000023ab3a3f x1846780607058624/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232505 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232480.539417:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607058624/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232505 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232480.539422:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232480.539424:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232480.539425:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058624:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232480.987103:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232482.011120:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232483.035089:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232484.059119:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232485.083110:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232485.659091:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232485.659106:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232485.659111:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232485.659124:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5358 00000100:00080000:1.0:1761232485.659128:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232485.659137:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232485.659198:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607058688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232485.659235:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232485.659240:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232485.659242:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232485.659246:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5358 00000100:00080000:1.0:1761232485.659248:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232485.659251:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232485.659261:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232485.659271:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607058752/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232485.659280:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232485.659284:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232485.659285:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232485.659290:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5358 00000100:00080000:1.0:1761232485.659292:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232485.659295:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232485.659299:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232485.659306:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607058816/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232485.659314:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232485.659316:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232485.659318:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232485.659321:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5358 00000100:00080000:1.0:1761232485.659323:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232485.659328:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232485.659331:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232485.659341:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607058880/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232485.659374:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607058688:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232485.659494:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232485/real 1761232485] req@00000000db10af3c x1846780607058688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232510 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232485.659515:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607058688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232510 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232485.659528:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232485.659533:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232485.659538:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607058688:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232485.659547:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058752:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232485.659574:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232485/real 1761232485] req@00000000659d78e3 x1846780607058752/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232510 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232485.659587:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607058752/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232510 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232485.659595:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232485.659598:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232485.659601:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058752:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232485.659607:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058816:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232485.659701:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232485/real 1761232485] req@00000000b72feabb x1846780607058816/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232510 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232485.659712:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607058816/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232510 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232485.659721:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232485.659723:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232485.659726:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058816:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232485.659731:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058880:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232485.659751:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232485/real 1761232485] req@00000000d07717c4 x1846780607058880/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232510 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232485.659768:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607058880/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232510 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232485.659776:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232485.659778:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232485.659780:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607058880:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232486.107128:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232487.131201:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232488.155079:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232489.179124:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232490.203101:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232490.779928:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232490.779938:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232490.779942:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232490.779952:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5363 00000100:00080000:3.0:1761232490.779955:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232490.779962:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232490.779996:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607058944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232490.780023:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232490.780027:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232490.780028:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232490.780031:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5363 00000100:00080000:3.0:1761232490.780033:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232490.780036:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232490.780044:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232490.780055:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607059008/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232490.780065:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761232490.780070:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607058944:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:3.0:1761232490.780072:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232490.780075:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232490.780082:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5363 00000100:00080000:3.0:1761232490.780084:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232490.780086:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232490.780090:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232490.780101:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607059072/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232490.780108:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232490.780111:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232490.780112:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232490.780115:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5363 00000100:00080000:3.0:1761232490.780117:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232490.780119:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232490.780122:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232490.780129:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607059136/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232490.780176:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232490/real 1761232490] req@00000000c7e4e432 x1846780607058944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232515 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232490.780198:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607058944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232515 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232490.780212:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232490.780216:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232490.780220:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607058944:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232490.780247:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059008:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232490.780274:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232490/real 1761232490] req@00000000f87d0618 x1846780607059008/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232515 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232490.780284:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607059008/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232515 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232490.780290:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232490.780292:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232490.780294:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059008:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232490.780298:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059072:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232490.780313:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232490/real 1761232490] req@000000007361665d x1846780607059072/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232515 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232490.780321:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607059072/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232515 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232490.780327:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232490.780329:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232490.780330:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059072:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232490.780334:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059136:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232490.780347:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232490/real 1761232490] req@0000000031e043f4 x1846780607059136/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232515 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232490.780355:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607059136/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232515 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232490.780362:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232490.780363:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232490.780365:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059136:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232491.227045:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232492.251091:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232493.275208:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232494.299062:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232495.323129:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232495.899048:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232495.899074:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232495.899080:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232495.899098:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5368 00000100:00080000:3.0:1761232495.899104:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232495.899115:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232495.899182:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607059200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232495.899225:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232495.899230:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232495.899232:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232495.899237:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5368 00000100:00080000:3.0:1761232495.899240:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232495.899243:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232495.899256:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232495.899269:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607059264/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232495.899279:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232495.899283:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232495.899285:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232495.899290:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5368 00000100:00080000:3.0:1761232495.899292:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232495.899296:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232495.899301:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232495.899316:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607059328/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232495.899329:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232495.899334:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232495.899336:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232495.899344:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5368 00000100:00080000:3.0:1761232495.899348:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232495.899354:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232495.899359:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232495.899371:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607059392/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232495.899387:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607059200:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232495.899555:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232495/real 1761232495] req@0000000083f16a6b x1846780607059200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232520 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232495.899578:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607059200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232520 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232495.899591:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232495.899595:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232495.899600:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607059200:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232495.899609:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059264:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232495.899640:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232495/real 1761232495] req@00000000a58ca11e x1846780607059264/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232520 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232495.899653:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607059264/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232520 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232495.899661:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232495.899663:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232495.899666:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059264:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232495.899673:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059328:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232495.899697:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232495/real 1761232495] req@000000001f8e688e x1846780607059328/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232520 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232495.899724:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607059328/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232520 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232495.899735:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232495.899738:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232495.899741:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059328:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232495.899782:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059392:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232495.899810:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232495/real 1761232495] req@0000000090b40c66 x1846780607059392/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232520 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232495.899822:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607059392/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232520 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232495.899881:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232495.899884:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232495.899888:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059392:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232496.347038:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232497.371171:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232498.395069:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232499.419088:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232500.443086:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232501.019033:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232501.019046:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232501.019052:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232501.019063:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5373 00000100:00080000:0.0:1761232501.019068:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232501.019076:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232501.019124:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607059456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232501.019167:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232501.019175:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232501.019177:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232501.019185:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5373 00000100:00080000:0.0:1761232501.019189:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232501.019196:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232501.019211:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232501.019234:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607059520/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232501.019245:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232501.019250:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232501.019252:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232501.019257:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5373 00000100:00080000:0.0:1761232501.019260:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232501.019264:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232501.019271:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232501.019282:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607059584/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232501.019294:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232501.019298:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232501.019300:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232501.019305:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607059456:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761232501.019310:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5373 00000100:00080000:0.0:1761232501.019320:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232501.019324:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232501.019329:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232501.019341:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607059648/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232501.019500:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232501/real 1761232501] req@00000000271a9d8e x1846780607059456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232526 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232501.019525:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607059456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232526 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232501.019538:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232501.019543:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232501.019548:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607059456:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232501.019557:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059520:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232501.019597:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232501/real 1761232501] req@000000000243251f x1846780607059520/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232526 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232501.019610:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607059520/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232526 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232501.019618:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232501.019621:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232501.019624:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059520:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232501.019654:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059584:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232501.019679:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232501/real 1761232501] req@0000000099514377 x1846780607059584/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232526 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232501.019700:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607059584/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232526 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232501.019709:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232501.019711:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232501.019714:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059584:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232501.019720:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059648:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232501.019740:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232501/real 1761232501] req@0000000001b2f813 x1846780607059648/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232526 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232501.019759:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607059648/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232526 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232501.019796:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232501.019798:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232501.019802:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059648:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232501.467108:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232502.491128:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232503.515102:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232504.539118:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232505.563168:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232506.139074:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232506.139088:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232506.139093:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232506.139105:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5378 00000100:00080000:2.0:1761232506.139110:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232506.139118:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232506.139162:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607059712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232506.139201:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232506.139206:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232506.139209:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232506.139213:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5378 00000100:00080000:2.0:1761232506.139216:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232506.139219:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232506.139229:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232506.139241:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607059776/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232506.139254:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232506.139259:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232506.139264:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232506.139269:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5378 00000100:00080000:2.0:1761232506.139271:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232506.139275:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232506.139282:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232506.139296:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607059840/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232506.139310:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232506.139314:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232506.139317:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232506.139326:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607059712:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:2.0:1761232506.139326:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5378 00000100:00080000:2.0:1761232506.139338:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232506.139347:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232506.139355:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232506.139372:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607059904/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232506.139525:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232506/real 1761232506] req@0000000091106a97 x1846780607059712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232531 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232506.139552:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607059712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232531 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232506.139567:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232506.139572:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232506.139578:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607059712:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232506.139587:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059776:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232506.139616:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232506/real 1761232506] req@0000000094135f44 x1846780607059776/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232531 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232506.139629:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607059776/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232531 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232506.139637:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232506.139640:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232506.139643:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059776:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232506.139684:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059840:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232506.139710:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232506/real 1761232506] req@0000000064978143 x1846780607059840/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232531 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232506.139723:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607059840/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232531 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232506.139732:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232506.139734:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232506.139737:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059840:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232506.139743:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059904:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232506.139763:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232506/real 1761232506] req@00000000c069b612 x1846780607059904/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232531 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232506.139775:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607059904/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232531 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232506.139784:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232506.139786:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232506.139789:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607059904:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232506.587176:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232507.611065:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232508.635111:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232509.659135:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232510.683145:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232511.259065:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232511.259080:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232511.259085:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232511.259097:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5383 00000100:00080000:0.0:1761232511.259102:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232511.259111:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232511.259156:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607059968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232511.259199:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232511.259204:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232511.259206:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232511.259211:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5383 00000100:00080000:0.0:1761232511.259213:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232511.259217:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232511.259227:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232511.259239:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607060032/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232511.259250:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232511.259254:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232511.259257:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232511.259261:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5383 00000100:00080000:0.0:1761232511.259263:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232511.259267:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232511.259271:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232511.259283:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607060096/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232511.259291:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232511.259295:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232511.259297:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232511.259302:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5383 00000100:00080000:0.0:1761232511.259304:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232511.259309:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232511.259313:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232511.259356:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607060160/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232511.259390:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607059968:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232511.259505:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232511/real 1761232511] req@000000007f60e210 x1846780607059968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232536 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232511.259527:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607059968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232536 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232511.259541:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232511.259545:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232511.259549:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607059968:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232511.259557:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060032:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232511.259584:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232511/real 1761232511] req@00000000fcce404c x1846780607060032/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232536 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232511.259596:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607060032/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232536 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232511.259604:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232511.259607:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232511.259610:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060032:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232511.259615:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060096:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232511.259636:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232511/real 1761232511] req@00000000b8f2be78 x1846780607060096/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232536 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232511.259648:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607060096/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232536 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232511.259658:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232511.259660:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232511.259663:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060096:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232511.259668:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060160:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232511.259689:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232511/real 1761232511] req@0000000001b2f813 x1846780607060160/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232536 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232511.259700:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607060160/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232536 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232511.259710:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232511.259712:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232511.259714:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060160:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232511.707111:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232512.731177:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232513.755119:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232514.779162:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232515.803007:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232516.379042:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232516.379056:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232516.379060:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232516.379074:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5388 00000100:00080000:1.0:1761232516.379079:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232516.379087:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232516.379134:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607060224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232516.379172:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232516.379177:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232516.379179:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232516.379183:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5388 00000100:00080000:1.0:1761232516.379186:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232516.379189:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232516.379199:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232516.379212:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607060288/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232516.379224:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232516.379227:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232516.379229:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232516.379233:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5388 00000100:00080000:1.0:1761232516.379235:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232516.379238:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232516.379242:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232516.379251:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607060352/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232516.379280:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232516.379284:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232516.379285:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232516.379290:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5388 00000100:00080000:1.0:1761232516.379293:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232516.379296:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232516.379301:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232516.379312:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607060416/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232516.379393:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607060224:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232516.379572:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232516/real 1761232516] req@00000000d07717c4 x1846780607060224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232541 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232516.379611:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607060224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232541 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232516.379632:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232516.379639:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232516.379647:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607060224:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232516.379662:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060288:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232516.379720:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232516/real 1761232516] req@00000000b72feabb x1846780607060288/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232541 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232516.379744:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607060288/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232541 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232516.379759:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232516.379764:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232516.379770:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060288:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232516.379782:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060352:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232516.379889:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232516/real 1761232516] req@00000000659d78e3 x1846780607060352/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232541 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232516.379905:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607060352/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232541 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232516.379915:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232516.379919:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232516.379922:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060352:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232516.379930:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060416:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232516.379960:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232516/real 1761232516] req@00000000db10af3c x1846780607060416/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232541 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232516.379972:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607060416/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232541 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232516.379982:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232516.379984:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232516.379987:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060416:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232516.827193:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232517.851190:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232518.875132:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232519.899136:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232520.923164:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232521.499005:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232521.499014:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232521.499017:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232521.499026:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5393 00000100:00080000:3.0:1761232521.499029:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232521.499035:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232521.499065:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607060480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232521.499090:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232521.499093:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232521.499094:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232521.499097:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5393 00000100:00080000:3.0:1761232521.499099:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232521.499101:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232521.499107:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232521.499114:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607060544/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232521.499120:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232521.499122:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232521.499122:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232521.499125:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5393 00000100:00080000:3.0:1761232521.499126:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232521.499127:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232521.499130:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232521.499135:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607060608/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232521.499139:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232521.499140:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232521.499141:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232521.499143:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5393 00000100:00080000:3.0:1761232521.499144:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232521.499146:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232521.499149:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232521.499154:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607060672/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232521.499177:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607060480:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232521.499266:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060544:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232521.499294:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060608:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232521.499318:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060672:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232521.947168:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232522.971092:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232523.995286:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761232524.571352:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232521/real 1761232524] req@000000006572b591 x1846780607060480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232546 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232524.571389:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607060480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232546 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232524.571402:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232524.571407:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232524.571412:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607060480:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232524.571422:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232521/real 1761232524] req@00000000fc01c41c x1846780607060544/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232546 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232524.571443:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607060544/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232546 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232524.571455:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232524.571460:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232524.571464:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060544:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232524.571474:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232521/real 1761232524] req@00000000408f088c x1846780607060608/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232546 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232524.571491:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607060608/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232546 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232524.571505:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232524.571510:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232524.571514:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060608:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232524.571524:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232521/real 1761232524] req@000000002ce934d3 x1846780607060672/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232546 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232524.571543:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607060672/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232546 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232524.571556:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232524.571560:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232524.571564:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060672:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232525.019107:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232526.043223:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232526.619063:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232526.619075:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232526.619079:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232526.619089:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5399 00000100:00080000:1.0:1761232526.619092:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232526.619099:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232526.619132:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607060736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232526.619179:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232526.619183:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232526.619184:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232526.619188:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5399 00000100:00080000:1.0:1761232526.619190:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232526.619193:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232526.619201:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232526.619212:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607060800/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232526.619221:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232526.619225:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232526.619226:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232526.619229:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5399 00000100:00080000:1.0:1761232526.619231:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232526.619236:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1761232526.619239:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607060736:192.168.201.104@tcp:250:kworker/u8:2.0 00000008:00100000:1.0:1761232526.619242:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232526.619259:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607060864/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232526.619272:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232526.619276:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232526.619277:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232526.619281:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5399 00000100:00080000:1.0:1761232526.619284:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232526.619288:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232526.619293:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232526.619306:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607060928/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232526.619372:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232526/real 1761232526] req@00000000db10af3c x1846780607060736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232551 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232526.619402:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607060736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232551 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232526.619418:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232526.619424:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232526.619450:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607060736:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232526.619472:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060800:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232526.619529:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232526/real 1761232526] req@00000000659d78e3 x1846780607060800/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232551 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232526.619552:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607060800/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232551 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232526.619565:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232526.619571:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232526.619577:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060800:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232526.619617:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060864:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232526.619677:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232526/real 1761232526] req@00000000b72feabb x1846780607060864/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232551 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232526.619698:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607060864/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232551 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232526.619714:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232526.619718:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232526.619724:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060864:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232526.619735:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060928:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232526.619783:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232526/real 1761232526] req@00000000d07717c4 x1846780607060928/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232551 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232526.619804:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607060928/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232551 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232526.619817:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232526.619822:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232526.619868:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607060928:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232527.067185:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232528.091184:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232529.115101:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232530.139060:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232531.163125:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232531.739116:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232531.739131:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232531.739136:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232531.739149:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5404 00000100:00080000:1.0:1761232531.739153:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232531.739163:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232531.739211:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607060992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232531.739250:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232531.739255:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232531.739257:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232531.739262:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5404 00000100:00080000:1.0:1761232531.739264:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232531.739268:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232531.739278:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232531.739290:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607061056/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232531.739301:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232531.739304:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232531.739306:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232531.739310:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5404 00000100:00080000:1.0:1761232531.739312:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232531.739315:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232531.739319:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232531.739328:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607061120/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232531.739339:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232531.739343:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232531.739345:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232531.739349:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5404 00000100:00080000:1.0:1761232531.739352:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232531.739357:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232531.739363:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232531.739377:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607061184/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232531.739415:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607060992:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232531.739601:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232531/real 1761232531] req@00000000d07717c4 x1846780607060992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232556 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232531.739628:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607060992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232556 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232531.739643:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232531.739650:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232531.739655:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607060992:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232531.739666:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061056:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232531.739701:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232531/real 1761232531] req@00000000b72feabb x1846780607061056/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232556 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232531.739714:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607061056/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232556 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232531.739722:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232531.739725:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232531.739728:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061056:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232531.739734:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061120:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232531.739757:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232531/real 1761232531] req@00000000659d78e3 x1846780607061120/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232556 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232531.739768:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607061120/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232556 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232531.739777:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232531.739779:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232531.739782:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061120:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232531.739821:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061184:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232531.739910:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232531/real 1761232531] req@00000000db10af3c x1846780607061184/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232556 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232531.739932:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607061184/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232556 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232531.739943:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232531.739945:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232531.739949:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061184:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232532.187227:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232533.211081:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232534.235174:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232535.259188:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232536.283211:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232536.859086:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232536.859101:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232536.859106:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232536.859119:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5409 00000100:00080000:0.0:1761232536.859123:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232536.859131:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232536.859176:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607061248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232536.859212:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232536.859217:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232536.859218:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232536.859223:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5409 00000100:00080000:0.0:1761232536.859226:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232536.859229:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232536.859239:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232536.859254:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607061312/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232536.859265:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232536.859268:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232536.859270:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232536.859274:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5409 00000100:00080000:0.0:1761232536.859276:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232536.859279:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232536.859283:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232536.859294:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607061376/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232536.859305:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232536.859310:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232536.859314:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232536.859318:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5409 00000100:00080000:0.0:1761232536.859320:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232536.859324:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232536.859331:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232536.859344:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607061440/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232536.859371:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607061248:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232536.859537:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232536/real 1761232536] req@0000000099514377 x1846780607061248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232561 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232536.859561:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607061248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232561 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232536.859577:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232536.859582:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232536.859588:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607061248:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232536.859597:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061312:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232536.859628:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232536/real 1761232536] req@000000000243251f x1846780607061312/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232561 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232536.859641:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607061312/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232561 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232536.859650:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232536.859652:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232536.859655:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061312:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232536.859664:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061376:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232536.859687:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232536/real 1761232536] req@00000000271a9d8e x1846780607061376/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232561 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232536.859699:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607061376/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232561 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232536.859709:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232536.859712:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232536.859714:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061376:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232536.859755:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061440:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232536.859779:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232536/real 1761232536] req@0000000023ab3a3f x1846780607061440/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232561 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232536.859790:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607061440/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232561 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232536.859799:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232536.859802:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232536.859805:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061440:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232537.307152:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232538.331144:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232539.355188:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232540.379147:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232541.403121:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232541.979001:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232541.979016:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232541.979021:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232541.979034:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5414 00000100:00080000:0.0:1761232541.979038:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232541.979046:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232541.979091:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607061504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232541.979124:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232541.979129:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232541.979131:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232541.979136:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5414 00000100:00080000:0.0:1761232541.979138:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232541.979142:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232541.979152:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232541.979164:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607061568/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232541.979174:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232541.979177:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232541.979179:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232541.979183:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5414 00000100:00080000:0.0:1761232541.979185:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232541.979188:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232541.979192:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232541.979203:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607061632/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232541.979214:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232541.979218:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232541.979220:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232541.979224:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5414 00000100:00080000:0.0:1761232541.979226:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232541.979229:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232541.979234:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232541.979250:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607061696/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232541.979252:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607061504:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232541.979368:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232541/real 1761232541] req@000000005a096598 x1846780607061504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232566 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232541.979389:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607061504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232566 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232541.979404:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232541.979408:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232541.979412:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607061504:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232541.979420:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061568:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232541.979447:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232541/real 1761232541] req@00000000275aeee5 x1846780607061568/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232566 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232541.979460:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607061568/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232566 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232541.979468:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232541.979470:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232541.979473:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061568:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232541.979480:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061632:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232541.979501:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232541/real 1761232541] req@000000002ef1f54f x1846780607061632/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232566 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232541.979513:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607061632/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232566 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232541.979522:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232541.979524:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232541.979527:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061632:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232541.979557:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061696:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232541.979582:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232541/real 1761232541] req@0000000023ab3a3f x1846780607061696/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232566 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232541.979594:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607061696/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232566 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232541.979603:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232541.979605:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232541.979608:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061696:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232542.427156:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232543.451181:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232544.475187:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232545.499197:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232546.523019:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232547.098997:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232547.099009:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232547.099014:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232547.099025:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5419 00000100:00080000:0.0:1761232547.099028:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232547.099035:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232547.099070:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607061760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232547.099096:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232547.099101:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232547.099102:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232547.099107:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5419 00000100:00080000:0.0:1761232547.099110:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232547.099115:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232547.099126:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232547.099130:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607061760:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761232547.099142:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607061824/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232547.099154:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232547.099158:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232547.099161:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232547.099165:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5419 00000100:00080000:0.0:1761232547.099167:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232547.099171:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232547.099176:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232547.099189:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607061888/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232547.099199:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232547.099203:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232547.099206:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761232547.099209:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232547/real 1761232547] req@00000000271a9d8e x1846780607061760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232572 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:0.0:1761232547.099213:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5419 00000100:00080000:0.0:1761232547.099219:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232547.099225:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1761232547.099228:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607061760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232572 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000008:00100000:0.0:1761232547.099231:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00080000:3.0:1761232547.099239:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232547.099242:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232547.099244:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607061760:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761232547.099250:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607061952/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232547.099266:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061824:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232547.099285:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232547/real 1761232547] req@000000000243251f x1846780607061824/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232572 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232547.099292:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607061824/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232572 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232547.099296:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232547.099298:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232547.099300:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061824:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232547.099303:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061888:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232547.099315:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232547/real 1761232547] req@0000000099514377 x1846780607061888/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232572 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232547.099321:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607061888/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232572 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232547.099326:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232547.099327:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232547.099328:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061888:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232547.099331:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061952:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232547.099340:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232547/real 1761232547] req@0000000001b2f813 x1846780607061952/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232572 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232547.099346:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607061952/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232572 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232547.099351:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232547.099352:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232547.099353:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607061952:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232547.547054:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232548.571166:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232549.595114:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232550.619122:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232551.643283:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232552.219124:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232552.219138:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232552.219143:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232552.219157:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5424 00000100:00080000:1.0:1761232552.219162:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232552.219172:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232552.219219:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607062016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232552.219256:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232552.219261:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232552.219263:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232552.219268:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5424 00000100:00080000:1.0:1761232552.219270:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232552.219273:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232552.219283:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232552.219295:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607062080/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232552.219306:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232552.219309:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232552.219311:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232552.219316:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5424 00000100:00080000:1.0:1761232552.219319:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232552.219322:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232552.219328:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232552.219340:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607062144/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232552.219350:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232552.219354:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232552.219357:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232552.219364:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5424 00000100:00080000:1.0:1761232552.219368:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1761232552.219374:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607062016:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761232552.219379:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232552.219393:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232552.219418:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607062208/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232552.219493:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232552/real 1761232552] req@00000000db10af3c x1846780607062016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232577 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232552.219523:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607062016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232577 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232552.219535:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232552.219538:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232552.219542:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607062016:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232552.219551:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062080:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232552.219578:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232552/real 1761232552] req@00000000659d78e3 x1846780607062080/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232577 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232552.219590:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607062080/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232577 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232552.219599:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232552.219601:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232552.219604:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062080:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232552.219610:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062144:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232552.219631:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232552/real 1761232552] req@00000000b72feabb x1846780607062144/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232577 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232552.219643:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607062144/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232577 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232552.219652:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232552.219655:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232552.219657:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062144:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232552.219688:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062208:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232552.219711:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232552/real 1761232552] req@00000000d07717c4 x1846780607062208/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232577 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232552.219722:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607062208/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232577 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232552.219731:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232552.219733:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232552.219736:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062208:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232552.667171:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232553.691142:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232554.715148:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232555.739219:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232556.763021:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232557.338953:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232557.338962:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232557.338965:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232557.338974:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5429 00000100:00080000:0.0:1761232557.338977:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232557.338983:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232557.339014:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607062272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232557.339037:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232557.339039:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232557.339041:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232557.339044:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5429 00000100:00080000:0.0:1761232557.339045:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232557.339048:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232557.339055:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232557.339065:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607062336/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232557.339073:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761232557.339076:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607062272:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761232557.339078:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232557.339081:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232557.339088:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5429 00000100:00080000:0.0:1761232557.339090:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232557.339092:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232557.339096:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232557.339105:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607062400/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232557.339111:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232557.339114:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232557.339115:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232557.339117:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5429 00000100:00080000:0.0:1761232557.339118:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232557.339120:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232557.339123:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232557.339130:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607062464/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232557.339163:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232557/real 1761232557] req@00000000b8f2be78 x1846780607062272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232582 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232557.339177:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607062272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232582 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232557.339185:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232557.339187:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232557.339191:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607062272:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232557.339223:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062336:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232557.339244:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232557/real 1761232557] req@00000000fcce404c x1846780607062336/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232582 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232557.339251:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607062336/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232582 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232557.339261:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232557.339263:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232557.339265:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062336:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232557.339269:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062400:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232557.339282:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232557/real 1761232557] req@000000007f60e210 x1846780607062400/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232582 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232557.339288:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607062400/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232582 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232557.339294:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232557.339295:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232557.339296:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062400:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232557.339300:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062464:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232557.339312:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232557/real 1761232557] req@0000000001b2f813 x1846780607062464/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232582 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232557.339318:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607062464/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232582 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232557.339323:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232557.339324:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232557.339326:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062464:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232557.787137:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232558.811205:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232559.835141:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232560.859133:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232561.883163:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232562.458951:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232562.458963:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232562.458967:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232562.458980:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5434 00000100:00080000:2.0:1761232562.458985:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232562.458993:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232562.459037:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607062528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232562.459062:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232562.459065:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232562.459066:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232562.459070:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5434 00000100:00080000:2.0:1761232562.459072:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232562.459076:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232562.459085:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232562.459094:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607062592/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232562.459100:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232562.459104:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232562.459106:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232562.459109:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5434 00000100:00080000:2.0:1761232562.459110:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232562.459112:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232562.459116:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232562.459124:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607062656/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232562.459129:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232562.459131:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232562.459132:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232562.459135:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5434 00000100:00080000:2.0:1761232562.459136:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1761232562.459139:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607062528:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:2.0:1761232562.459141:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232562.459148:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232562.459154:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607062720/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232562.459243:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232562/real 1761232562] req@00000000f38f27df x1846780607062528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232587 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232562.459258:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607062528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232587 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232562.459266:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232562.459269:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232562.459272:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607062528:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232562.459277:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062592:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232562.459293:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232562/real 1761232562] req@0000000017386078 x1846780607062592/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232587 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232562.459299:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607062592/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232587 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232562.459304:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232562.459305:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232562.459306:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062592:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232562.459328:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062656:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232562.459342:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232562/real 1761232562] req@00000000c069b612 x1846780607062656/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232587 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232562.459348:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607062656/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232587 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232562.459353:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232562.459355:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232562.459357:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062656:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232562.459360:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062720:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232562.459370:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232562/real 1761232562] req@0000000064978143 x1846780607062720/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232587 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232562.459376:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607062720/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232587 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232562.459381:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232562.459383:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232562.459384:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062720:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232562.907177:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232563.931153:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232564.955120:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232565.979136:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232567.003069:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232567.579084:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232567.579096:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232567.579101:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232567.579112:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5439 00000100:00080000:1.0:1761232567.579116:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232567.579124:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232567.579166:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607062784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232567.579199:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232567.579203:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232567.579205:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232567.579209:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5439 00000100:00080000:1.0:1761232567.579212:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232567.579215:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232567.579224:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232567.579235:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607062848/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232567.579244:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232567.579248:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232567.579249:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232567.579252:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5439 00000100:00080000:1.0:1761232567.579254:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232567.579256:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232567.579260:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232567.579269:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607062912/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232567.579279:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232567.579283:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232567.579285:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232567.579288:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5439 00000100:00080000:1.0:1761232567.579290:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232567.579293:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232567.579297:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232567.579307:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607062976/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232567.579313:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607062784:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232567.579418:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232567/real 1761232567] req@00000000d07717c4 x1846780607062784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232592 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232567.579437:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607062784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232592 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232567.579448:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232567.579452:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232567.579456:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607062784:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232567.579464:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062848:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232567.579487:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232567/real 1761232567] req@00000000b72feabb x1846780607062848/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232592 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232567.579505:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607062848/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232592 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232567.579513:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232567.579515:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232567.579517:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062848:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232567.579522:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062912:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232567.579541:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232567/real 1761232567] req@00000000659d78e3 x1846780607062912/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232592 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232567.579550:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607062912/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232592 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232567.579558:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232567.579560:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232567.579563:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062912:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232567.579590:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062976:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232567.579610:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232567/real 1761232567] req@00000000db10af3c x1846780607062976/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232592 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232567.579619:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607062976/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232592 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232567.579628:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232567.579630:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232567.579633:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607062976:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232568.027194:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232569.051119:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232570.075083:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232571.099107:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232572.123181:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232572.698989:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232572.698997:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232572.699001:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232572.699009:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5445 00000100:00080000:3.0:1761232572.699012:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232572.699018:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232572.699053:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607063040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232572.699075:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232572.699078:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232572.699079:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232572.699082:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5445 00000100:00080000:3.0:1761232572.699083:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232572.699086:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232572.699092:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232572.699101:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607063104/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232572.699108:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761232572.699109:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607063040:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:3.0:1761232572.699113:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232572.699116:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232572.699119:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5445 00000100:00080000:3.0:1761232572.699120:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232572.699121:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232572.699126:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232572.699134:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607063168/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232572.699140:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232572.699142:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232572.699143:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232572.699145:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5445 00000100:00080000:3.0:1761232572.699146:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232572.699148:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232572.699151:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232572.699156:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607063232/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232572.699189:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232572/real 1761232572] req@000000002ce934d3 x1846780607063040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232597 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232572.699201:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607063040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232597 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232572.699209:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232572.699211:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232572.699214:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607063040:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232572.699230:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063104:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232572.699246:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232572/real 1761232572] req@00000000408f088c x1846780607063104/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232597 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232572.699252:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607063104/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232597 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232572.699256:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232572.699258:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232572.699259:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063104:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232572.699262:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063168:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232572.699274:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232572/real 1761232572] req@00000000fc01c41c x1846780607063168/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232597 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232572.699284:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607063168/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232597 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232572.699289:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232572.699290:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232572.699291:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063168:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232572.699294:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063232:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232572.699304:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232572/real 1761232572] req@000000006572b591 x1846780607063232/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232597 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232572.699310:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607063232/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232597 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232572.699315:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232572.699316:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232572.699317:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063232:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232573.147218:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232574.171144:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232575.195156:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232576.219087:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232577.243173:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232577.818922:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232577.818934:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232577.818938:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232577.818949:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5450 00000100:00080000:3.0:1761232577.818953:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232577.818960:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232577.818998:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607063296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232577.819025:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232577.819029:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232577.819031:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232577.819034:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5450 00000100:00080000:3.0:1761232577.819037:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232577.819040:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232577.819048:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232577.819058:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607063360/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232577.819067:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232577.819070:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232577.819072:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232577.819075:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5450 00000100:00080000:3.0:1761232577.819077:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232577.819080:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232577.819083:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232577.819094:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607063296:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232577.819098:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607063424/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232577.819111:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232577.819113:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232577.819114:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232577.819117:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5450 00000100:00080000:3.0:1761232577.819119:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232577.819122:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232577.819126:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232577.819135:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607063488/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232577.819195:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232577/real 1761232577] req@0000000090b40c66 x1846780607063296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232602 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232577.819212:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607063296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232602 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232577.819222:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232577.819225:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232577.819228:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607063296:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232577.819236:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063360:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232577.819256:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232577/real 1761232577] req@000000001f8e688e x1846780607063360/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232602 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232577.819266:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607063360/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232602 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232577.819272:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232577.819274:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232577.819276:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063360:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232577.819300:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063424:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232577.819318:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232577/real 1761232577] req@00000000a58ca11e x1846780607063424/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232602 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232577.819327:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607063424/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232602 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232577.819334:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232577.819336:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232577.819338:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063424:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232577.819342:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063488:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232577.819357:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232577/real 1761232577] req@0000000083f16a6b x1846780607063488/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232602 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232577.819366:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607063488/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232602 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232577.819373:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232577.819375:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232577.819378:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063488:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232578.267124:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232579.291194:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232580.314974:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232581.339046:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232582.363080:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232582.938918:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232582.938931:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232582.938936:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232582.938949:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5455 00000100:00080000:3.0:1761232582.938953:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232582.938962:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232582.939009:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607063552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232582.939039:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232582.939044:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232582.939045:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232582.939050:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5455 00000100:00080000:3.0:1761232582.939052:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232582.939055:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232582.939068:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232582.939081:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607063616/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232582.939090:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232582.939093:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232582.939095:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232582.939098:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5455 00000100:00080000:3.0:1761232582.939101:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232582.939104:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1761232582.939108:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607063552:192.168.201.104@tcp:250:kworker/u8:1.0 00000008:00100000:3.0:1761232582.939108:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232582.939119:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607063680/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232582.939128:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232582.939131:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232582.939133:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232582.939137:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5455 00000100:00080000:3.0:1761232582.939139:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232582.939142:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232582.939147:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232582.939155:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607063744/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232582.939234:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232582/real 1761232582] req@0000000031e043f4 x1846780607063552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232607 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232582.939249:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607063552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232607 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232582.939258:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232582.939260:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232582.939263:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607063552:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232582.939268:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063616:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232582.939290:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232582/real 1761232582] req@000000007361665d x1846780607063616/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232607 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232582.939297:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607063616/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232607 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232582.939301:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232582.939302:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232582.939303:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063616:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232582.939328:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063680:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232582.939343:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232582/real 1761232582] req@00000000f87d0618 x1846780607063680/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232607 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232582.939350:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607063680/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232607 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232582.939355:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232582.939356:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232582.939358:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063680:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232582.939360:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063744:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232582.939371:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232582/real 1761232582] req@00000000c7e4e432 x1846780607063744/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232607 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232582.939378:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607063744/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232607 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232582.939383:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232582.939384:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232582.939385:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063744:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232583.387166:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232584.411137:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232585.435056:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232586.459196:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232587.483128:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232588.059071:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232588.059085:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232588.059089:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232588.059101:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5460 00000100:00080000:0.0:1761232588.059105:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232588.059113:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232588.059164:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607063808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232588.059210:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232588.059217:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232588.059219:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232588.059227:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5460 00000100:00080000:0.0:1761232588.059231:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232588.059237:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232588.059250:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232588.059275:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607063872/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232588.059288:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607063808:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761232588.059302:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232588.059311:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232588.059314:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232588.059322:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5460 00000100:00080000:0.0:1761232588.059327:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232588.059333:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232588.059343:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232588.059364:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607063936/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232588.059376:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232588.059380:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232588.059382:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232588.059386:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5460 00000100:00080000:0.0:1761232588.059388:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232588.059392:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232588.059398:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232588.059411:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607064000/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232588.059418:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063872:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232588.059485:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063936:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232588.059512:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064000:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232588.507128:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232589.531157:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232590.555074:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761232591.131432:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232588/real 1761232591] req@0000000023ab3a3f x1846780607064000/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232613 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232591.131476:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607064000/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232613 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232591.131496:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232591.131501:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232591.131509:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064000:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232591.131520:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232588/real 1761232591] req@000000000243251f x1846780607063872/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232613 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232591.131538:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607063872/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232613 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232591.131551:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232591.131555:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232591.131560:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063872:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232591.131570:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232588/real 1761232591] req@00000000271a9d8e x1846780607063936/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232613 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232591.131586:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607063936/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232613 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232591.131599:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232591.131603:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232591.131608:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607063936:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232591.131617:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232588/real 1761232591] req@0000000099514377 x1846780607063808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232613 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232591.131647:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607063808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232613 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232591.131659:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232591.131663:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232591.131667:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607063808:192.168.201.104@tcp:250:kworker/u8:1.0 00010000:00010000:0.0:1761232591.579104:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232592.603071:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232593.178967:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232593.178981:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232593.178985:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232593.178999:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5465 00000100:00080000:0.0:1761232593.179004:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232593.179013:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232593.179056:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607064064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232593.179089:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232593.179094:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232593.179096:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232593.179100:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5465 00000100:00080000:0.0:1761232593.179103:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232593.179106:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232593.179115:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232593.179130:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607064128/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232593.179140:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232593.179143:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232593.179145:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232593.179149:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5465 00000100:00080000:0.0:1761232593.179151:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232593.179154:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232593.179158:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232593.179168:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607064192/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232593.179176:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232593.179180:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232593.179182:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232593.179185:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5465 00000100:00080000:0.0:1761232593.179189:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232593.179194:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232593.179198:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232593.179208:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607064256/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232593.179287:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607064064:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232593.179516:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232593/real 1761232593] req@000000002ef1f54f x1846780607064064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232618 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232593.179551:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607064064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232618 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232593.179572:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232593.179579:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232593.179587:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607064064:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232593.179601:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064128:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232593.179659:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232593/real 1761232593] req@00000000275aeee5 x1846780607064128/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232618 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232593.179682:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607064128/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232618 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232593.179696:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232593.179701:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232593.179706:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064128:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232593.179719:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064192:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232593.179771:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232593/real 1761232593] req@000000005a096598 x1846780607064192/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232618 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232593.179793:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607064192/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232618 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232593.179809:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232593.179813:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232593.179819:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064192:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232593.179944:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064256:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232593.180000:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232593/real 1761232593] req@0000000099514377 x1846780607064256/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232618 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232593.180022:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607064256/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232618 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232593.180038:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232593.180043:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232593.180048:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064256:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232593.627156:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232594.651209:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232595.675157:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232596.699165:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232597.723206:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232598.298985:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232598.298998:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232598.299003:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232598.299016:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5470 00000100:00080000:0.0:1761232598.299021:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232598.299030:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232598.299081:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607064320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232598.299115:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232598.299121:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232598.299123:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232598.299129:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5470 00000100:00080000:0.0:1761232598.299132:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232598.299136:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232598.299147:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232598.299162:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607064384/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232598.299173:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232598.299176:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232598.299178:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232598.299182:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5470 00000100:00080000:0.0:1761232598.299184:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232598.299186:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232598.299191:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232598.299200:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607064448/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232598.299209:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232598.299212:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232598.299214:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232598.299218:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5470 00000100:00080000:0.0:1761232598.299221:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232598.299225:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232598.299229:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232598.299238:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607064512/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232598.299313:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607064320:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232598.299532:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232598/real 1761232598] req@00000000271a9d8e x1846780607064320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232623 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232598.299557:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607064320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232623 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232598.299575:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232598.299580:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232598.299585:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607064320:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232598.299595:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064384:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232598.299624:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232598/real 1761232598] req@000000000243251f x1846780607064384/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232623 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232598.299637:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607064384/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232623 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232598.299645:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232598.299647:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232598.299650:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064384:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232598.299656:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064448:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232598.299680:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232598/real 1761232598] req@0000000023ab3a3f x1846780607064448/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232623 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232598.299691:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607064448/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232623 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232598.299701:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232598.299704:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232598.299707:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064448:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232598.299713:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064512:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232598.299736:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232598/real 1761232598] req@0000000001b2f813 x1846780607064512/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232623 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232598.299748:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607064512/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232623 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232598.299757:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232598.299760:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232598.299762:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064512:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232598.747186:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232599.771226:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232600.795201:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232601.819140:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232602.843198:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232603.419010:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232603.419025:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232603.419031:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232603.419043:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5475 00000100:00080000:0.0:1761232603.419048:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232603.419056:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232603.419105:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607064576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232603.419143:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232603.419149:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232603.419153:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232603.419159:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5475 00000100:00080000:0.0:1761232603.419162:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232603.419166:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232603.419177:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232603.419193:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607064640/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232603.419204:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232603.419208:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232603.419211:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232603.419215:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5475 00000100:00080000:0.0:1761232603.419217:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232603.419220:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232603.419229:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232603.419248:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607064704/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232603.419259:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232603.419263:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232603.419265:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232603.419293:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5475 00000100:00080000:0.0:1761232603.419296:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232603.419299:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232603.419304:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232603.419318:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607064768/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232603.419347:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607064576:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232603.419466:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232603/real 1761232603] req@000000007f60e210 x1846780607064576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232628 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232603.419488:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607064576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232628 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232603.419501:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232603.419505:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232603.419510:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607064576:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232603.419518:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064640:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232603.419545:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232603/real 1761232603] req@00000000fcce404c x1846780607064640/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232628 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232603.419557:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607064640/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232628 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232603.419566:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232603.419569:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232603.419572:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064640:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232603.419577:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064704:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232603.419601:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232603/real 1761232603] req@00000000b8f2be78 x1846780607064704/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232628 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232603.419612:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607064704/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232628 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232603.419621:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232603.419623:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232603.419626:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064704:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232603.419656:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064768:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232603.419679:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232603/real 1761232603] req@0000000001b2f813 x1846780607064768/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232628 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232603.419691:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607064768/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232628 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232603.419705:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232603.419707:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232603.419710:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064768:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232603.867170:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232604.891219:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232605.915068:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232606.939214:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232607.963181:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232608.539021:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232608.539037:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232608.539042:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232608.539054:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5480 00000100:00080000:2.0:1761232608.539058:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232608.539067:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232608.539111:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607064832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232608.539154:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232608.539161:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232608.539164:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232608.539171:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5480 00000100:00080000:2.0:1761232608.539175:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232608.539181:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232608.539195:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232608.539244:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607064896/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232608.539265:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232608.539271:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232608.539276:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761232608.539278:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607064832:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761232608.539289:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5480 00000100:00080000:2.0:1761232608.539298:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232608.539304:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232608.539314:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232608.539339:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607064960/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232608.539355:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232608.539361:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232608.539363:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232608.539370:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5480 00000100:00080000:2.0:1761232608.539374:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232608.539380:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232608.539388:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232608.539410:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607065024/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232608.539422:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232608/real 1761232608] req@0000000094135f44 x1846780607064832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232633 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232608.539456:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607064832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232633 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232608.539474:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232608.539480:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232608.539487:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607064832:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232608.539524:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064896:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232608.539577:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232608/real 1761232608] req@0000000091106a97 x1846780607064896/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232633 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232608.539597:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607064896/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232633 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232608.539610:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232608.539614:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232608.539619:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064896:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232608.539629:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064960:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232608.539677:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232608/real 1761232608] req@0000000064978143 x1846780607064960/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232633 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232608.539697:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607064960/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232633 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232608.539711:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232608.539717:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232608.539722:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607064960:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232608.539732:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065024:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232608.539772:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232608/real 1761232608] req@00000000c069b612 x1846780607065024/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232633 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232608.539796:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607065024/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232633 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232608.539811:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232608.539815:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232608.539822:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065024:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232608.987111:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232610.011048:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232611.035116:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232612.058981:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232613.083036:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232613.659021:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232613.659035:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232613.659040:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232613.659053:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5486 00000100:00080000:0.0:1761232613.659057:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232613.659065:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232613.659108:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607065088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232613.659143:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232613.659149:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232613.659150:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232613.659176:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5486 00000100:00080000:0.0:1761232613.659179:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232613.659184:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232613.659195:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232613.659214:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607065152/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232613.659229:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232613.659234:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232613.659238:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761232613.659239:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607065088:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761232613.659249:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5486 00000100:00080000:0.0:1761232613.659254:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232613.659257:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232613.659263:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232613.659279:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607065216/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232613.659290:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232613.659294:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232613.659296:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232613.659300:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5486 00000100:00080000:0.0:1761232613.659303:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232613.659306:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232613.659310:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232613.659322:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607065280/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232613.659364:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232613/real 1761232613] req@0000000023ab3a3f x1846780607065088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232638 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232613.659387:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607065088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232638 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232613.659400:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232613.659404:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232613.659409:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607065088:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232613.659437:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065152:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232613.659467:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232613/real 1761232613] req@000000000243251f x1846780607065152/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232638 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232613.659489:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607065152/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232638 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232613.659497:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232613.659500:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232613.659503:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065152:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232613.659509:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065216:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232613.659532:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232613/real 1761232613] req@00000000271a9d8e x1846780607065216/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232638 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232613.659544:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607065216/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232638 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232613.659553:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232613.659555:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232613.659558:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065216:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232613.659563:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065280:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232613.659584:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232613/real 1761232613] req@0000000099514377 x1846780607065280/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232638 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232613.659595:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607065280/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232638 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232613.659604:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232613.659606:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232613.659610:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065280:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232614.107083:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232615.131172:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232616.155099:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232617.179025:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232618.203170:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232618.778919:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232618.778928:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232618.778932:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232618.778941:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5491 00000100:00080000:0.0:1761232618.778944:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232618.778950:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232618.778981:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607065344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232618.779003:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232618.779006:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232618.779007:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232618.779010:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5491 00000100:00080000:0.0:1761232618.779012:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232618.779013:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232618.779020:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232618.779027:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607065408/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232618.779042:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232618.779044:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232618.779045:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232618.779048:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5491 00000100:00080000:0.0:1761232618.779050:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232618.779053:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232618.779057:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232618.779066:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607065472/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232618.779071:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232618.779074:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232618.779074:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232618.779076:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5491 00000100:00080000:0.0:1761232618.779078:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232618.779080:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232618.779082:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232618.779088:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607065536/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232618.779139:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607065344:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232618.779212:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232618/real 1761232618] req@000000005a096598 x1846780607065344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232643 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232618.779224:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607065344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232643 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232618.779232:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232618.779234:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232618.779237:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607065344:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232618.779241:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065408:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232618.779255:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232618/real 1761232618] req@00000000275aeee5 x1846780607065408/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232643 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232618.779261:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607065408/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232643 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232618.779264:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232618.779266:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232618.779267:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065408:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232618.779270:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065472:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232618.779281:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232618/real 1761232618] req@000000002ef1f54f x1846780607065472/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232643 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232618.779287:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607065472/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232643 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232618.779292:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232618.779293:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232618.779295:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065472:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232618.779298:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065536:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232618.779308:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232618/real 1761232618] req@0000000099514377 x1846780607065536/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232643 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232618.779313:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607065536/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232643 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232618.779318:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232618.779319:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232618.779320:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065536:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232619.226993:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232620.251202:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232621.275037:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232622.299088:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232623.323189:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232623.898933:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232623.898943:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232623.898946:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232623.898956:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5496 00000100:00080000:0.0:1761232623.898959:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232623.898965:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232623.898998:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607065600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232623.899023:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232623.899026:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232623.899027:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232623.899030:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5496 00000100:00080000:0.0:1761232623.899032:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232623.899034:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232623.899058:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232623.899074:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607065664/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232623.899074:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607065600:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761232623.899100:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232623.899103:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232623.899104:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232623.899108:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5496 00000100:00080000:0.0:1761232623.899109:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232623.899112:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232623.899115:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232623.899126:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607065728/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232623.899132:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232623.899136:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232623.899137:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232623.899139:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5496 00000100:00080000:0.0:1761232623.899140:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232623.899143:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232623.899147:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232623.899155:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607065792/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232623.899176:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232623/real 1761232623] req@00000000271a9d8e x1846780607065600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232648 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232623.899191:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607065600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232648 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232623.899199:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232623.899201:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232623.899205:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607065600:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232623.899225:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065664:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232623.899246:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232623/real 1761232623] req@000000000243251f x1846780607065664/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232648 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232623.899254:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607065664/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232648 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232623.899259:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232623.899261:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232623.899262:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065664:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232623.899267:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065728:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232623.899280:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232623/real 1761232623] req@0000000023ab3a3f x1846780607065728/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232648 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232623.899287:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607065728/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232648 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232623.899293:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232623.899294:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232623.899296:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065728:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232623.899299:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065792:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232623.899311:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232623/real 1761232623] req@0000000001b2f813 x1846780607065792/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232648 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232623.899319:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607065792/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232648 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232623.899325:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232623.899327:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232623.899329:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065792:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232624.347167:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232625.371224:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232626.395104:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232627.419050:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232628.443182:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232629.019077:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232629.019091:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232629.019095:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232629.019109:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5501 00000100:00080000:1.0:1761232629.019113:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232629.019122:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232629.019170:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607065856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232629.019207:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232629.019214:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232629.019217:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232629.019225:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5501 00000100:00080000:1.0:1761232629.019228:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232629.019235:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232629.019249:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232629.019271:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607065920/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232629.019287:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607065856:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761232629.019290:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232629.019296:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232629.019299:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232629.019304:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5501 00000100:00080000:1.0:1761232629.019308:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232629.019313:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232629.019320:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232629.019335:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607065984/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232629.019347:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232629.019352:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232629.019355:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232629.019359:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5501 00000100:00080000:1.0:1761232629.019362:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232629.019367:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232629.019372:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232629.019386:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607066048/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232629.019406:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232629/real 1761232629] req@00000000db10af3c x1846780607065856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232654 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232629.019434:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607065856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232654 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232629.019450:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232629.019455:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232629.019462:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607065856:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232629.019497:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065920:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232629.019537:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232629/real 1761232629] req@00000000659d78e3 x1846780607065920/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232654 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232629.019551:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607065920/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232654 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232629.019560:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232629.019562:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232629.019566:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065920:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232629.019572:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065984:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232629.019593:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232629/real 1761232629] req@00000000b72feabb x1846780607065984/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232654 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232629.019613:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607065984/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232654 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232629.019622:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232629.019624:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232629.019626:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607065984:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232629.019632:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066048:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232629.019651:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232629/real 1761232629] req@00000000d07717c4 x1846780607066048/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232654 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232629.019662:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607066048/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232654 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232629.019670:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232629.019672:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232629.019675:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066048:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232629.467187:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232630.491210:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232631.515123:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232632.539197:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232633.563159:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232634.138907:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232634.138915:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232634.138918:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232634.138938:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5506 00000100:00080000:1.0:1761232634.138940:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232634.138946:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232634.138979:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607066112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232634.138998:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232634.139001:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232634.139002:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232634.139005:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5506 00000100:00080000:1.0:1761232634.139007:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232634.139008:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232634.139014:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232634.139021:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607066176/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232634.139026:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232634.139028:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232634.139029:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232634.139031:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5506 00000100:00080000:1.0:1761232634.139032:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232634.139034:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232634.139036:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232634.139040:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607066240/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232634.139045:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232634.139047:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232634.139048:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232634.139051:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5506 00000100:00080000:1.0:1761232634.139053:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232634.139055:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232634.139057:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232634.139063:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607066304/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232634.139071:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607066112:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232634.139144:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232634/real 1761232634] req@00000000d07717c4 x1846780607066112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232659 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232634.139155:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607066112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232659 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232634.139161:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232634.139164:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232634.139166:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607066112:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232634.139171:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066176:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232634.139185:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232634/real 1761232634] req@00000000b72feabb x1846780607066176/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232659 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232634.139192:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607066176/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232659 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232634.139196:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232634.139198:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232634.139199:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066176:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232634.139202:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066240:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232634.139213:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232634/real 1761232634] req@00000000659d78e3 x1846780607066240/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232659 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232634.139218:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607066240/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232659 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232634.139224:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232634.139225:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232634.139226:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066240:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232634.139245:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066304:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232634.139256:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232634/real 1761232634] req@00000000db10af3c x1846780607066304/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232659 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232634.139263:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607066304/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232659 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232634.139268:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232634.139269:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232634.139270:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066304:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232634.587016:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232635.610975:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232636.635165:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232637.659131:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232638.683106:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232639.259043:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232639.259055:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232639.259059:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232639.259071:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5511 00000100:00080000:1.0:1761232639.259075:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232639.259083:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232639.259128:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607066368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232639.259161:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232639.259165:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232639.259166:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232639.259171:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5511 00000100:00080000:1.0:1761232639.259173:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232639.259176:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232639.259185:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232639.259196:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607066432/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232639.259207:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232639.259212:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232639.259214:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232639.259218:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5511 00000100:00080000:1.0:1761232639.259221:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232639.259224:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232639.259227:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232639.259239:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607066496/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232639.259250:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232639.259254:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232639.259256:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761232639.259266:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607066368:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761232639.259273:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5511 00000100:00080000:1.0:1761232639.259281:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232639.259287:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232639.259292:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232639.259309:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607066560/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232639.259445:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232639/real 1761232639] req@00000000db10af3c x1846780607066368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232664 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232639.259483:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607066368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232664 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232639.259504:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232639.259511:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232639.259519:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607066368:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232639.259535:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066432:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232639.259587:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232639/real 1761232639] req@00000000659d78e3 x1846780607066432/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232664 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232639.259603:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607066432/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232664 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232639.259612:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232639.259616:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232639.259619:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066432:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232639.259659:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066496:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232639.259688:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232639/real 1761232639] req@00000000b72feabb x1846780607066496/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232664 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232639.259701:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607066496/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232664 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232639.259710:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232639.259713:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232639.259715:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066496:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232639.259721:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066560:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232639.259742:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232639/real 1761232639] req@00000000d07717c4 x1846780607066560/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232664 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232639.259753:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607066560/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232664 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232639.259763:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232639.259766:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232639.259768:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066560:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232639.707162:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232640.731147:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232641.755136:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232642.779084:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232643.803039:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232644.379082:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232644.379100:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232644.379106:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232644.379124:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5516 00000100:00080000:3.0:1761232644.379129:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232644.379140:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232644.379194:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607066624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232644.379246:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232644.379254:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232644.379256:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232644.379264:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5516 00000100:00080000:3.0:1761232644.379269:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232644.379277:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232644.379292:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232644.379314:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607066624:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232644.379321:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607066688/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232644.379346:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232644.379353:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232644.379357:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232644.379364:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5516 00000100:00080000:3.0:1761232644.379369:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232644.379376:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232644.379386:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232644.379408:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607066752/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232644.379425:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232644.379432:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232644.379435:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232644.379445:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232644/real 1761232644] req@0000000086ccb527 x1846780607066624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232669 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232644.379447:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5516 00000100:00080000:3.0:1761232644.379456:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232644.379470:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232644.379483:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232644.379486:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607066624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232669 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232644.379504:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1761232644.379511:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607066816/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232644.379513:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232644.379525:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607066624:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232644.379569:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066688:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232644.379607:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232644/real 1761232644] req@00000000bd684ccc x1846780607066688/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232669 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232644.379622:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607066688/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232669 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232644.379631:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232644.379634:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232644.379638:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066688:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232644.379643:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066752:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232644.379665:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232644/real 1761232644] req@0000000018a8f5da x1846780607066752/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232669 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232644.379677:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607066752/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232669 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232644.379686:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232644.379689:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232644.379691:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066752:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232644.379697:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066816:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232644.379716:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232644/real 1761232644] req@000000005a9bdefd x1846780607066816/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232669 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232644.379736:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607066816/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232669 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232644.379745:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232644.379747:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232644.379750:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066816:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232644.827138:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232645.851129:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232646.875149:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232647.899115:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232648.923118:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232649.498999:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232649.499012:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232649.499016:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232649.499028:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5521 00000100:00080000:0.0:1761232649.499031:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232649.499040:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232649.499085:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607066880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232649.499113:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232649.499116:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232649.499117:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232649.499121:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5521 00000100:00080000:0.0:1761232649.499123:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232649.499126:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232649.499135:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232649.499148:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607066944/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232649.499155:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232649.499158:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232649.499160:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232649.499167:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607066880:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761232649.499168:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5521 00000100:00080000:0.0:1761232649.499175:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232649.499182:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232649.499187:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232649.499195:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607067008/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232649.499201:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232649.499203:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232649.499205:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232649.499208:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5521 00000100:00080000:0.0:1761232649.499209:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232649.499211:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232649.499214:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232649.499225:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607067072/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232649.499276:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232649/real 1761232649] req@00000000b8f2be78 x1846780607066880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232674 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232649.499293:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607066880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232674 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232649.499303:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232649.499307:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232649.499310:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607066880:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232649.499334:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066944:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232649.499355:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232649/real 1761232649] req@00000000fcce404c x1846780607066944/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232674 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232649.499364:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607066944/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232674 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232649.499370:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232649.499372:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232649.499374:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607066944:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232649.499380:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067008:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232649.499396:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232649/real 1761232649] req@000000007f60e210 x1846780607067008/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232674 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232649.499405:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607067008/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232674 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232649.499412:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232649.499414:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232649.499416:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067008:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232649.499420:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067072:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232649.499434:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232649/real 1761232649] req@0000000001b2f813 x1846780607067072/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232674 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232649.499441:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607067072/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232674 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232649.499448:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232649.499450:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232649.499452:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067072:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232649.947156:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232650.971171:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232651.995088:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232653.019179:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232654.043113:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232654.618987:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232654.618999:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232654.619003:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232654.619012:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5527 00000100:00080000:0.0:1761232654.619016:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232654.619022:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232654.619058:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607067136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232654.619093:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232654.619098:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232654.619100:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232654.619106:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5527 00000100:00080000:0.0:1761232654.619108:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232654.619112:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232654.619122:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232654.619135:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607067200/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232654.619145:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232654.619150:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232654.619152:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232654.619158:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5527 00000100:00080000:0.0:1761232654.619161:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232654.619166:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232654.619175:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232654.619210:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607067136:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:0.0:1761232654.619211:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607067264/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232654.619238:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232654.619243:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232654.619246:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232654.619253:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5527 00000100:00080000:0.0:1761232654.619257:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232654.619262:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232654.619269:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232654.619289:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607067328/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232654.619384:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067200:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232654.619452:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067264:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232654.619509:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067328:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232655.067017:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232656.090977:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232657.115088:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761232657.691464:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232654/real 1761232657] req@00000000271a9d8e x1846780607067264/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232679 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232657.691501:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607067264/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232679 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232657.691516:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232657.691520:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232657.691526:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067264:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232657.691533:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232654/real 1761232657] req@0000000099514377 x1846780607067328/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232679 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232657.691545:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607067328/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232679 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232657.691553:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232657.691556:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232657.691559:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067328:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232657.691565:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232654/real 1761232657] req@0000000023ab3a3f x1846780607067136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232679 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232657.691577:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607067136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232679 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232657.691586:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232657.691588:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232657.691591:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607067136:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232657.691597:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232654/real 1761232657] req@000000000243251f x1846780607067200/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232679 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232657.691609:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607067200/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232679 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232657.691617:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232657.691620:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232657.691622:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067200:192.168.201.104@tcp:38:kworker/u8:1.0 00010000:00010000:2.0:1761232658.139118:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232659.163023:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232659.739035:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232659.739048:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232659.739052:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232659.739066:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5532 00000100:00080000:0.0:1761232659.739070:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232659.739078:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232659.739116:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607067392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232659.739143:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232659.739146:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232659.739148:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232659.739152:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5532 00000100:00080000:0.0:1761232659.739155:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232659.739159:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232659.739168:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232659.739179:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607067456/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232659.739188:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232659.739192:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232659.739193:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232659.739196:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5532 00000100:00080000:0.0:1761232659.739198:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232659.739201:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232659.739205:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232659.739213:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607067520/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232659.739220:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232659.739223:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232659.739225:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232659.739228:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5532 00000100:00080000:0.0:1761232659.739231:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232659.739235:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232659.739239:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232659.739248:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607067584/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232659.739269:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607067392:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232659.739375:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232659/real 1761232659] req@000000002ef1f54f x1846780607067392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232684 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232659.739402:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607067392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232684 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232659.739418:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232659.739422:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232659.739428:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607067392:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232659.739438:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067456:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232659.739482:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232659/real 1761232659] req@00000000275aeee5 x1846780607067456/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232684 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232659.739501:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607067456/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232684 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232659.739512:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232659.739516:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232659.739521:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067456:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232659.739531:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067520:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232659.739586:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232659/real 1761232659] req@000000005a096598 x1846780607067520/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232684 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232659.739604:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607067520/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232684 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232659.739618:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232659.739621:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232659.739626:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067520:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232659.739702:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067584:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232659.739746:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232659/real 1761232659] req@000000000243251f x1846780607067584/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232684 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232659.739765:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607067584/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232684 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232659.739779:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232659.739782:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232659.739786:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067584:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232660.187105:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232661.211104:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232662.235041:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232663.259163:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232664.283142:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232664.858996:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232664.859008:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232664.859012:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232664.859023:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5537 00000100:00080000:3.0:1761232664.859027:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232664.859052:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232664.859100:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607067648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232664.859149:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232664.859154:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232664.859156:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232664.859161:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5537 00000100:00080000:3.0:1761232664.859163:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232664.859167:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232664.859177:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232664.859189:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607067712/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232664.859209:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232664.859213:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232664.859215:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232664.859220:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5537 00000100:00080000:3.0:1761232664.859222:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232664.859226:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232664.859230:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232664.859255:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607067776/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232664.859265:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761232664.859293:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607067648:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761232664.859309:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232664.859315:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232664.859326:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5537 00000100:00080000:3.0:1761232664.859330:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232664.859337:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232664.859348:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232664.859371:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607067840/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232664.859415:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232664/real 1761232664] req@00000000daa39bb1 x1846780607067648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232689 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232664.859452:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607067648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232689 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232664.859467:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232664.859471:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232664.859476:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607067648:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232664.859484:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067712:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232664.859512:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232664/real 1761232664] req@000000003ad149cc x1846780607067712/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232689 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232664.859524:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607067712/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232689 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232664.859532:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232664.859535:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232664.859538:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067712:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232664.859544:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067776:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232664.859565:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232664/real 1761232664] req@0000000088473b3e x1846780607067776/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232689 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232664.859633:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607067776/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232689 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232664.859643:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232664.859646:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232664.859649:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067776:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232664.859683:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067840:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232664.859712:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232664/real 1761232664] req@000000004e3f612a x1846780607067840/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232689 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232664.859725:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607067840/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232689 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232664.859733:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232664.859736:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232664.859739:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067840:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232665.307051:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232666.331031:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232667.355084:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232668.379121:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232669.403083:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232669.979085:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232669.979100:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232669.979104:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232669.979117:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5542 00000100:00080000:1.0:1761232669.979122:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232669.979131:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232669.979179:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607067904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232669.979222:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232669.979227:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232669.979229:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232669.979234:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5542 00000100:00080000:1.0:1761232669.979236:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232669.979240:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232669.979251:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232669.979264:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607067968/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232669.979276:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232669.979280:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232669.979281:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232669.979285:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5542 00000100:00080000:1.0:1761232669.979287:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232669.979290:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232669.979294:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232669.979304:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607068032/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232669.979314:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232669.979318:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232669.979320:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232669.979327:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5542 00000100:00080000:1.0:1761232669.979331:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232669.979335:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232669.979341:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232669.979351:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607068096/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232669.979393:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607067904:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232669.979636:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232669/real 1761232669] req@00000000d07717c4 x1846780607067904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232694 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232669.979675:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607067904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232694 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232669.979697:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232669.979705:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232669.979714:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607067904:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232669.979737:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067968:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232669.979801:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232669/real 1761232669] req@00000000b72feabb x1846780607067968/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232694 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232669.979874:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607067968/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232694 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232669.979892:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232669.979897:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232669.979905:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607067968:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232669.979917:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068032:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232669.979981:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232669/real 1761232669] req@00000000659d78e3 x1846780607068032/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232694 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232669.980007:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607068032/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232694 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232669.980024:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232669.980029:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232669.980036:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068032:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232669.980097:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068096:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232669.980156:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232669/real 1761232669] req@00000000db10af3c x1846780607068096/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232694 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232669.980181:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607068096/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232694 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232669.980197:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232669.980202:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232669.980207:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068096:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232670.427166:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232671.451100:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232672.475090:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232673.499005:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232674.522973:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232675.099076:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232675.099089:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232675.099094:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232675.099107:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5547 00000100:00080000:2.0:1761232675.099112:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232675.099121:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232675.099166:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607068160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232675.099202:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232675.099207:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232675.099209:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232675.099214:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5547 00000100:00080000:2.0:1761232675.099217:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232675.099220:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232675.099230:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232675.099243:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607068224/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232675.099254:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232675.099257:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232675.099259:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232675.099263:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5547 00000100:00080000:2.0:1761232675.099265:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232675.099268:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232675.099273:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232675.099287:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607068288/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232675.099297:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232675.099302:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232675.099304:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232675.099309:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5547 00000100:00080000:2.0:1761232675.099311:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232675.099315:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232675.099320:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232675.099325:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607068160:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761232675.099337:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607068352/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232675.099441:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232675/real 1761232675] req@0000000017386078 x1846780607068160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232700 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232675.099488:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607068160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232700 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232675.099502:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232675.099505:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232675.099510:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607068160:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232675.099519:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068224:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232675.099550:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232675/real 1761232675] req@00000000f38f27df x1846780607068224/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232700 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232675.099563:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607068224/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232700 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232675.099571:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232675.099573:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232675.099576:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068224:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232675.099582:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068288:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232675.099606:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232675/real 1761232675] req@00000000c069b612 x1846780607068288/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232700 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232675.099618:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607068288/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232700 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232675.099627:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232675.099629:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232675.099631:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068288:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232675.099661:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068352:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232675.099683:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232675/real 1761232675] req@0000000064978143 x1846780607068352/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232700 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232675.099694:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607068352/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232700 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232675.099703:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232675.099705:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232675.099707:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068352:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232675.547140:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232676.571013:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232677.595106:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232678.619189:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232679.643207:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232680.219038:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232680.219052:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232680.219057:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232680.219090:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5552 00000100:00080000:3.0:1761232680.219094:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232680.219104:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232680.219158:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607068416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232680.219199:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232680.219204:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232680.219206:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232680.219211:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5552 00000100:00080000:3.0:1761232680.219213:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232680.219217:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232680.219226:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232680.219239:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607068480/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232680.219250:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232680.219254:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232680.219256:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232680.219260:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5552 00000100:00080000:3.0:1761232680.219262:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232680.219265:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232680.219269:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232680.219278:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607068544/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232680.219286:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232680.219289:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232680.219291:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232680.219294:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5552 00000100:00080000:3.0:1761232680.219298:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232680.219302:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232680.219307:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232680.219317:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607068608/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232680.219447:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607068416:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232680.219614:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232680/real 1761232680] req@000000004e3f612a x1846780607068416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232705 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232680.219639:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607068416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232705 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232680.219656:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232680.219661:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232680.219666:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607068416:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232680.219676:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068480:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232680.219704:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232680/real 1761232680] req@0000000088473b3e x1846780607068480/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232705 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232680.219717:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607068480/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232705 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232680.219725:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232680.219728:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232680.219731:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068480:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232680.219737:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068544:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232680.219760:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232680/real 1761232680] req@000000003ad149cc x1846780607068544/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232705 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232680.219773:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607068544/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232705 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232680.219782:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232680.219784:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232680.219787:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068544:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232680.219793:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068608:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232680.219815:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232680/real 1761232680] req@00000000daa39bb1 x1846780607068608/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232705 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232680.219902:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607068608/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232705 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232680.219913:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232680.219916:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232680.219919:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068608:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232680.667244:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232681.691210:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232682.715123:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232683.739271:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232684.763034:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232685.338980:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232685.338992:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232685.338996:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232685.339009:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5557 00000100:00080000:1.0:1761232685.339012:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232685.339019:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232685.339064:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607068672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232685.339089:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232685.339092:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232685.339093:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232685.339097:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5557 00000100:00080000:1.0:1761232685.339098:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232685.339100:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232685.339108:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232685.339116:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607068736/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232685.339124:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232685.339127:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232685.339128:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232685.339130:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5557 00000100:00080000:1.0:1761232685.339131:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232685.339135:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1761232685.339137:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607068672:192.168.201.104@tcp:250:kworker/u8:2.0 00000008:00100000:1.0:1761232685.339139:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232685.339148:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607068800/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232685.339152:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232685.339155:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232685.339156:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232685.339158:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5557 00000100:00080000:1.0:1761232685.339159:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232685.339161:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232685.339164:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232685.339172:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607068864/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232685.339258:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232685/real 1761232685] req@00000000db10af3c x1846780607068672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232710 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232685.339278:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607068672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232710 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232685.339287:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232685.339289:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232685.339292:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607068672:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232685.339314:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068736:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232685.339336:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232685/real 1761232685] req@00000000659d78e3 x1846780607068736/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232710 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232685.339361:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607068736/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232710 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232685.339366:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232685.339368:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232685.339370:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068736:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232685.339374:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068800:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232685.339394:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232685/real 1761232685] req@00000000b72feabb x1846780607068800/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232710 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232685.339401:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607068800/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232710 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232685.339407:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232685.339408:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232685.339410:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068800:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232685.339413:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068864:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232685.339425:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232685/real 1761232685] req@00000000d07717c4 x1846780607068864/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232710 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232685.339431:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607068864/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232710 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232685.339436:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232685.339438:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232685.339439:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068864:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232685.787044:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232686.811207:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232687.835189:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232688.859111:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232689.883106:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232690.458972:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232690.458984:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232690.458988:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232690.459000:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5562 00000100:00080000:3.0:1761232690.459004:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232690.459012:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232690.459051:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607068928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232690.459085:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232690.459089:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232690.459091:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232690.459095:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5562 00000100:00080000:3.0:1761232690.459098:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232690.459100:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232690.459110:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232690.459120:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607068992/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232690.459130:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232690.459133:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232690.459134:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232690.459138:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5562 00000100:00080000:3.0:1761232690.459139:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232690.459142:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232690.459146:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232690.459154:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607069056/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232690.459163:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232690.459167:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232690.459169:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232690.459173:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5562 00000100:00080000:3.0:1761232690.459175:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232690.459179:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232690.459183:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232690.459195:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607068928:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232690.459196:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607069120/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232690.459350:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232690/real 1761232690] req@000000005a9bdefd x1846780607068928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232715 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232690.459369:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607068928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232715 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232690.459381:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232690.459384:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232690.459388:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607068928:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232690.459395:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068992:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232690.459443:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232690/real 1761232690] req@0000000018a8f5da x1846780607068992/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232715 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232690.459454:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607068992/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232715 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232690.459462:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232690.459465:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232690.459467:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607068992:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232690.459473:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069056:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232690.459494:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232690/real 1761232690] req@00000000bd684ccc x1846780607069056/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232715 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232690.459504:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607069056/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232715 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232690.459512:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232690.459514:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232690.459516:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069056:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232690.459543:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069120:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232690.459564:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232690/real 1761232690] req@0000000086ccb527 x1846780607069120/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232715 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232690.459574:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607069120/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232715 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232690.459582:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232690.459583:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232690.459586:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069120:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232690.907048:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232691.931118:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232692.955130:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232693.979148:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232695.002986:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232695.578980:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232695.578989:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232695.578993:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232695.579002:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5567 00000100:00080000:1.0:1761232695.579004:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232695.579010:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232695.579043:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607069184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232695.579066:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232695.579069:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232695.579070:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232695.579073:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5567 00000100:00080000:1.0:1761232695.579074:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232695.579076:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232695.579083:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232695.579091:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607069248/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232695.579099:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232695.579102:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1761232695.579103:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607069184:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761232695.579104:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232695.579111:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5567 00000100:00080000:1.0:1761232695.579112:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232695.579114:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232695.579117:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232695.579124:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607069312/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232695.579129:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232695.579132:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232695.579133:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232695.579135:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5567 00000100:00080000:1.0:1761232695.579136:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232695.579138:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232695.579141:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232695.579147:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607069376/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232695.579183:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232695/real 1761232695] req@00000000d07717c4 x1846780607069184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232720 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232695.579196:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607069184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232720 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232695.579204:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232695.579206:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232695.579209:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607069184:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232695.579256:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069248:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232695.579275:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232695/real 1761232695] req@00000000b72feabb x1846780607069248/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232720 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232695.579282:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607069248/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232720 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232695.579287:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232695.579288:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232695.579290:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069248:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232695.579293:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069312:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232695.579307:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232695/real 1761232695] req@00000000659d78e3 x1846780607069312/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232720 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232695.579314:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607069312/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232720 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232695.579318:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232695.579319:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232695.579321:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069312:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232695.579323:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069376:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232695.579335:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232695/real 1761232695] req@00000000db10af3c x1846780607069376/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232720 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232695.579341:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607069376/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232720 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232695.579345:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232695.579346:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232695.579348:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069376:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232696.026985:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232697.050984:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232698.074981:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232699.099013:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232700.123045:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232700.698992:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232700.699002:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232700.699005:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232700.699013:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5573 00000100:00080000:0.0:1761232700.699016:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232700.699022:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232700.699053:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607069440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232700.699075:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232700.699078:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232700.699079:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232700.699082:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5573 00000100:00080000:0.0:1761232700.699083:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232700.699085:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232700.699091:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232700.699097:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607069504/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232700.699104:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232700.699106:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232700.699107:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232700.699110:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5573 00000100:00080000:0.0:1761232700.699111:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232700.699114:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232700.699117:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232700.699122:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607069440:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761232700.699125:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607069568/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232700.699133:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232700.699135:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232700.699135:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232700.699138:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5573 00000100:00080000:0.0:1761232700.699139:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232700.699141:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232700.699144:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232700.699151:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607069632/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232700.699213:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232700/real 1761232700] req@0000000023ab3a3f x1846780607069440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232725 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232700.699226:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607069440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232725 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232700.699233:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232700.699236:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232700.699238:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607069440:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232700.699243:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069504:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232700.699259:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232700/real 1761232700] req@0000000099514377 x1846780607069504/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232725 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232700.699266:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607069504/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232725 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232700.699270:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232700.699271:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232700.699273:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069504:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232700.699290:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069568:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232700.699303:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232700/real 1761232700] req@00000000271a9d8e x1846780607069568/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232725 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232700.699310:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607069568/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232725 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232700.699315:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232700.699316:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232700.699317:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069568:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232700.699320:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069632:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232700.699330:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232700/real 1761232700] req@0000000001b2f813 x1846780607069632/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232725 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232700.699341:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607069632/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232725 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232700.699346:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232700.699347:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232700.699348:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069632:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232701.147011:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232702.170999:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232703.195000:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232704.219061:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232705.243024:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232705.819074:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232705.819091:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232705.819097:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232705.819165:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5578 00000100:00080000:1.0:1761232705.819170:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232705.819182:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232705.819242:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607069696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232705.819286:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232705.819291:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232705.819293:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232705.819298:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5578 00000100:00080000:1.0:1761232705.819301:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232705.819304:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232705.819316:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232705.819329:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607069760/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232705.819342:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232705.819358:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232705.819360:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232705.819364:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5578 00000100:00080000:1.0:1761232705.819366:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232705.819372:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232705.819377:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232705.819391:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607069824/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232705.819391:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607069696:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761232705.819417:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232705.819421:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232705.819423:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232705.819430:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5578 00000100:00080000:1.0:1761232705.819432:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232705.819436:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232705.819440:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232705.819455:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607069888/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232705.819515:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232705/real 1761232705] req@00000000db10af3c x1846780607069696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232730 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232705.819530:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607069696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232730 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232705.819539:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232705.819542:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232705.819545:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607069696:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232705.819551:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069760:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232705.819570:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232705/real 1761232705] req@00000000659d78e3 x1846780607069760/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232730 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232705.819578:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607069760/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232730 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232705.819582:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232705.819584:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232705.819586:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069760:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232705.819610:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069824:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232705.819624:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232705/real 1761232705] req@00000000b72feabb x1846780607069824/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232730 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232705.819631:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607069824/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232730 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232705.819637:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232705.819638:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232705.819640:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069824:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232705.819644:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069888:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232705.819656:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232705/real 1761232705] req@00000000d07717c4 x1846780607069888/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232730 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232705.819663:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607069888/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232730 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232705.819668:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232705.819670:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232705.819671:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607069888:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232706.267166:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232707.291112:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232708.315028:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232709.339151:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232710.363011:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232710.939024:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232710.939033:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232710.939036:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232710.939069:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5583 00000100:00080000:3.0:1761232710.939072:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232710.939080:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232710.939130:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607069952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232710.939154:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232710.939157:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232710.939158:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232710.939161:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5583 00000100:00080000:3.0:1761232710.939162:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232710.939165:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232710.939172:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232710.939180:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607070016/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232710.939187:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232710.939189:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232710.939190:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232710.939193:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5583 00000100:00080000:3.0:1761232710.939194:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1761232710.939199:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607069952:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:3.0:1761232710.939199:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232710.939204:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232710.939211:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607070080/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232710.939218:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232710.939220:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232710.939221:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232710.939224:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5583 00000100:00080000:3.0:1761232710.939226:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232710.939228:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232710.939231:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232710.939238:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607070144/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232710.939280:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232710/real 1761232710] req@00000000c7e4e432 x1846780607069952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232735 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232710.939293:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607069952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232735 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232710.939300:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232710.939302:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232710.939305:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607069952:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232710.939310:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070016:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232710.939326:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232710/real 1761232710] req@00000000f87d0618 x1846780607070016/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232735 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232710.939333:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607070016/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232735 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232710.939337:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232710.939338:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232710.939340:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070016:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232710.939360:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070080:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232710.939373:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232710/real 1761232710] req@000000007361665d x1846780607070080/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232735 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232710.939379:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607070080/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232735 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232710.939385:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232710.939386:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232710.939387:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070080:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232710.939391:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070144:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232710.939402:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232710/real 1761232710] req@0000000031e043f4 x1846780607070144/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232735 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232710.939408:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607070144/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232735 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232710.939413:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232710.939414:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232710.939416:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070144:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232711.387082:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232712.411001:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232713.435013:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232714.459032:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232715.483039:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232716.058958:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232716.058967:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232716.058971:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232716.058980:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5588 00000100:00080000:0.0:1761232716.058983:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232716.059005:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232716.059046:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607070208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232716.059073:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232716.059077:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232716.059078:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232716.059082:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5588 00000100:00080000:0.0:1761232716.059084:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232716.059088:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232716.059095:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232716.059106:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607070272/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232716.059113:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232716.059117:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1761232716.059118:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607070208:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761232716.059119:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232716.059126:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5588 00000100:00080000:0.0:1761232716.059128:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232716.059130:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232716.059133:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232716.059140:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607070336/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232716.059145:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232716.059148:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232716.059149:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232716.059150:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5588 00000100:00080000:0.0:1761232716.059151:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232716.059153:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232716.059156:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232716.059162:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607070400/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232716.059197:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232716/real 1761232716] req@000000007f60e210 x1846780607070208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232741 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232716.059210:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607070208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232741 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232716.059217:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232716.059219:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232716.059222:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607070208:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232716.059239:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070272:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232716.059255:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232716/real 1761232716] req@00000000fcce404c x1846780607070272/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232741 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232716.059261:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607070272/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232741 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232716.059265:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232716.059266:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232716.059268:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070272:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232716.059271:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070336:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232716.059283:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232716/real 1761232716] req@00000000b8f2be78 x1846780607070336/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232741 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232716.059288:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607070336/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232741 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232716.059293:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232716.059294:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232716.059296:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070336:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232716.059298:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070400:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232716.059310:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232716/real 1761232716] req@0000000001b2f813 x1846780607070400/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232741 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232716.059316:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607070400/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232741 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232716.059320:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232716.059321:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232716.059323:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070400:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232716.507033:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232717.531028:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232718.554996:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232719.579056:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232720.603071:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232721.178903:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232721.178914:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232721.178918:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232721.178960:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5593 00000100:00080000:0.0:1761232721.178964:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232721.178971:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232721.179006:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607070464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232721.179033:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232721.179037:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232721.179039:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232721.179043:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5593 00000100:00080000:0.0:1761232721.179045:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232721.179049:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232721.179057:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232721.179069:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607070528/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232721.179079:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232721.179083:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232721.179084:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232721.179088:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5593 00000100:00080000:0.0:1761232721.179091:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232721.179099:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1761232721.179099:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607070464:192.168.201.104@tcp:250:kworker/u8:2.0 00000008:00100000:0.0:1761232721.179105:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232721.179119:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607070592/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232721.179129:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232721.179133:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232721.179135:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232721.179138:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5593 00000100:00080000:0.0:1761232721.179141:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232721.179144:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232721.179149:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232721.179158:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607070656/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232721.179189:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070528:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232721.179217:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070592:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232721.179231:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070656:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232721.626989:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232722.651084:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232723.675059:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761232724.251210:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232721/real 1761232724] req@0000000023ab3a3f x1846780607070592/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232746 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232724.251242:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607070592/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232746 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232724.251255:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232724.251259:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232724.251264:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070592:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232724.251270:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232721/real 1761232724] req@000000000243251f x1846780607070656/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232746 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232724.251288:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607070656/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232746 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232724.251296:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232724.251299:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232724.251302:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070656:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232724.251307:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232721/real 1761232724] req@00000000271a9d8e x1846780607070464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232746 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232724.251318:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607070464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232746 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232724.251324:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232724.251327:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232724.251329:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607070464:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232724.251335:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232721/real 1761232724] req@0000000099514377 x1846780607070528/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232746 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232724.251349:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607070528/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232746 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232724.251356:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232724.251358:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232724.251361:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070528:192.168.201.104@tcp:38:kworker/u8:2.0 00010000:00010000:2.0:1761232724.699026:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232725.723157:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232726.299115:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232726.299150:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232726.299157:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232726.299173:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5598 00000100:00080000:3.0:1761232726.299178:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232726.299189:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232726.299248:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607070720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232726.299300:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232726.299307:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232726.299309:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232726.299316:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5598 00000100:00080000:3.0:1761232726.299319:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232726.299325:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232726.299341:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232726.299361:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607070784/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232726.299376:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232726.299385:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232726.299390:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232726.299397:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5598 00000100:00080000:3.0:1761232726.299402:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232726.299410:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232726.299419:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232726.299458:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607070720:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232726.299463:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607070848/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232726.299502:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232726.299508:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232726.299511:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232726.299519:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5598 00000100:00080000:3.0:1761232726.299524:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232726.299533:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232726.299541:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232726.299565:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607070912/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232726.299590:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232726/real 1761232726] req@0000000083f16a6b x1846780607070720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232751 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232726.299615:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607070720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232751 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232726.299635:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232726.299645:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232726.299654:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607070720:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232726.299671:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070784:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232726.299725:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232726/real 1761232726] req@00000000a58ca11e x1846780607070784/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232751 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232726.299746:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607070784/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232751 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232726.299760:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232726.299765:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232726.299770:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070784:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232726.299816:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070848:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232726.299967:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232726/real 1761232726] req@000000001f8e688e x1846780607070848/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232751 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232726.299991:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607070848/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232751 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232726.300006:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232726.300011:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232726.300016:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070848:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232726.300029:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070912:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232726.300077:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232726/real 1761232726] req@0000000090b40c66 x1846780607070912/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232751 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232726.300099:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607070912/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232751 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232726.300115:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232726.300120:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232726.300125:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607070912:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232726.747148:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232727.771099:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232728.795110:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232729.819091:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232730.843035:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232731.418943:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232731.418956:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232731.418961:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232731.418974:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5603 00000100:00080000:3.0:1761232731.418978:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232731.418986:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232731.419030:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607070976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232731.419064:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232731.419068:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232731.419070:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232731.419075:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5603 00000100:00080000:3.0:1761232731.419078:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232731.419081:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232731.419090:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232731.419102:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607071040/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232731.419112:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232731.419115:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232731.419116:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232731.419120:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5603 00000100:00080000:3.0:1761232731.419122:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232731.419125:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232731.419129:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232731.419140:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607071104/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232731.419148:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232731.419152:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232731.419153:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232731.419159:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5603 00000100:00080000:3.0:1761232731.419163:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232731.419171:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232731.419179:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232731.419201:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607070976:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232731.419202:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607071168/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232731.419315:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232731/real 1761232731] req@000000006572b591 x1846780607070976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232756 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232731.419334:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607070976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232756 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232731.419346:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232731.419350:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232731.419354:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607070976:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232731.419362:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071040:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232731.419387:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232731/real 1761232731] req@00000000fc01c41c x1846780607071040/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232756 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232731.419398:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607071040/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232756 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232731.419414:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232731.419416:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232731.419419:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071040:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232731.419425:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071104:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232731.419444:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232731/real 1761232731] req@00000000408f088c x1846780607071104/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232756 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232731.419455:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607071104/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232756 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232731.419463:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232731.419465:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232731.419467:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071104:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232731.419497:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071168:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232731.419518:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232731/real 1761232731] req@000000002ce934d3 x1846780607071168/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232756 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232731.419528:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607071168/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232756 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232731.419536:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232731.419538:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232731.419540:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071168:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232731.867097:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232732.891098:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232733.915084:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232734.939117:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232735.963124:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232736.538947:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232736.538957:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232736.538960:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232736.538970:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5608 00000100:00080000:3.0:1761232736.538973:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232736.538979:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232736.539033:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607071232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232736.539064:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232736.539067:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232736.539069:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232736.539072:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5608 00000100:00080000:3.0:1761232736.539074:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232736.539076:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232736.539084:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232736.539093:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607071296/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232736.539101:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232736.539103:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232736.539104:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232736.539109:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5608 00000100:00080000:3.0:1761232736.539111:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232736.539113:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232736.539117:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232736.539124:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607071360/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232736.539130:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232736.539133:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232736.539134:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232736.539138:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5608 00000100:00080000:3.0:1761232736.539140:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232736.539143:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232736.539146:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232736.539154:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607071424/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232736.539173:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607071232:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232736.539262:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232736/real 1761232736] req@000000002ce934d3 x1846780607071232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232761 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232736.539277:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607071232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232761 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232736.539286:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232736.539289:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232736.539292:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607071232:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232736.539298:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071296:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232736.539316:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232736/real 1761232736] req@00000000408f088c x1846780607071296/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232761 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232736.539324:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607071296/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232761 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232736.539330:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232736.539332:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232736.539334:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071296:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232736.539338:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071360:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232736.539353:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232736/real 1761232736] req@00000000fc01c41c x1846780607071360/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232761 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232736.539361:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607071360/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232761 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232736.539368:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232736.539370:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232736.539372:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071360:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232736.539397:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071424:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232736.539413:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232736/real 1761232736] req@000000006572b591 x1846780607071424/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232761 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232736.539421:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607071424/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232761 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232736.539427:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232736.539429:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232736.539431:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071424:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232736.987108:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232738.011065:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232739.035021:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232740.059073:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232741.083051:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232741.658967:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232741.658976:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232741.658979:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232741.658988:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5614 00000100:00080000:1.0:1761232741.658991:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232741.658996:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232741.659029:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607071488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232741.659053:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232741.659056:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232741.659057:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232741.659060:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5614 00000100:00080000:1.0:1761232741.659061:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232741.659064:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232741.659071:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232741.659079:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607071552/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232741.659086:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761232741.659087:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607071488:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761232741.659089:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232741.659091:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232741.659095:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5614 00000100:00080000:1.0:1761232741.659097:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232741.659098:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232741.659101:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232741.659111:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607071616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232741.659116:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232741.659117:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232741.659119:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232741.659121:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5614 00000100:00080000:1.0:1761232741.659123:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232741.659125:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232741.659128:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232741.659134:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607071680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232741.659173:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232741/real 1761232741] req@00000000d07717c4 x1846780607071488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232766 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232741.659189:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607071488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232766 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232741.659196:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232741.659198:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232741.659201:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607071488:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232741.659221:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071552:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232741.659239:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232741/real 1761232741] req@00000000b72feabb x1846780607071552/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232766 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232741.659245:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607071552/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232766 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232741.659250:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232741.659251:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232741.659253:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071552:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232741.659256:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071616:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232741.659268:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232741/real 1761232741] req@00000000659d78e3 x1846780607071616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232766 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232741.659273:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607071616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232766 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232741.659278:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232741.659279:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232741.659281:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071616:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232741.659284:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071680:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232741.659295:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232741/real 1761232741] req@00000000db10af3c x1846780607071680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232766 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232741.659300:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607071680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232766 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232741.659305:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232741.659307:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232741.659308:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071680:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232742.106994:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232743.131118:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232744.155131:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232745.179015:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232746.203064:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232746.778975:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232746.778985:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232746.778989:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232746.778999:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5619 00000100:00080000:1.0:1761232746.779002:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232746.779008:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232746.779044:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607071744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232746.779070:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232746.779074:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232746.779076:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232746.779081:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5619 00000100:00080000:1.0:1761232746.779083:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232746.779086:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232746.779096:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232746.779110:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607071808/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232746.779120:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232746.779124:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232746.779126:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232746.779130:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5619 00000100:00080000:1.0:1761232746.779132:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232746.779135:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:3.0:1761232746.779136:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607071744:192.168.201.104@tcp:250:kworker/u8:1.0 00000008:00100000:1.0:1761232746.779140:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232746.779151:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607071872/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232746.779160:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232746.779163:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232746.779179:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232746.779183:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5619 00000100:00080000:1.0:1761232746.779185:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232746.779189:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232746.779193:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232746.779222:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607071936/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232746.779252:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232746/real 1761232746] req@00000000db10af3c x1846780607071744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232771 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232746.779281:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607071744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232771 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232746.779299:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232746.779305:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232746.779311:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607071744:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232746.779324:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071808:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232746.779372:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232746/real 1761232746] req@00000000659d78e3 x1846780607071808/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232771 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232746.779386:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607071808/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232771 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232746.779395:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232746.779398:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232746.779401:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071808:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232746.779430:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071872:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232746.779458:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232746/real 1761232746] req@00000000b72feabb x1846780607071872/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232771 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232746.779469:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607071872/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232771 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232746.779478:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232746.779481:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232746.779483:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071872:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232746.779488:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071936:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232746.779508:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232746/real 1761232746] req@00000000d07717c4 x1846780607071936/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232771 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232746.779519:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607071936/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232771 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232746.779528:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232746.779530:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232746.779533:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607071936:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232747.227052:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232748.251109:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232749.275095:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232750.299080:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232751.323118:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232751.898950:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232751.898960:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232751.898963:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232751.898973:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5624 00000100:00080000:1.0:1761232751.898976:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232751.898982:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232751.899016:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607072000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232751.899040:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232751.899043:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232751.899044:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232751.899047:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5624 00000100:00080000:1.0:1761232751.899049:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232751.899051:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232751.899057:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232751.899065:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607072064/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232751.899072:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232751.899091:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232751.899093:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232751.899097:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5624 00000100:00080000:1.0:1761232751.899099:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232751.899103:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232751.899106:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232751.899115:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607072128/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232751.899123:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232751.899126:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232751.899128:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232751.899134:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5624 00000100:00100000:3.0:1761232751.899134:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607072000:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761232751.899138:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232751.899144:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232751.899150:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232751.899159:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607072192/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232751.899252:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232751/real 1761232751] req@00000000d07717c4 x1846780607072000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232776 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232751.899277:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607072000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232776 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232751.899290:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232751.899296:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232751.899302:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607072000:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232751.899311:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072064:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232751.899352:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232751/real 1761232751] req@00000000b72feabb x1846780607072064/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232776 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232751.899369:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607072064/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232776 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232751.899379:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232751.899382:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232751.899387:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072064:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232751.899421:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072128:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232751.899458:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232751/real 1761232751] req@00000000659d78e3 x1846780607072128/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232776 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232751.899494:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607072128/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232776 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232751.899509:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232751.899513:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232751.899518:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072128:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232751.899538:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072192:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232751.899660:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232751/real 1761232751] req@00000000db10af3c x1846780607072192/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232776 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232751.899686:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607072192/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232776 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232751.899696:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232751.899698:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232751.899702:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072192:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232752.346990:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232753.371029:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232754.395145:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232755.419230:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232756.443102:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232757.018894:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232757.018904:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232757.018908:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232757.018918:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5629 00000100:00080000:3.0:1761232757.018922:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232757.018928:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232757.018962:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607072256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232757.018986:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232757.018991:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232757.018992:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232757.018996:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5629 00000100:00080000:3.0:1761232757.018999:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232757.019003:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232757.019011:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232757.019023:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607072256:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232757.019027:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607072320/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232757.019038:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232757.019041:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232757.019042:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232757.019046:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5629 00000100:00080000:3.0:1761232757.019047:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232757.019049:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232757.019053:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232757.019070:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607072384/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232757.019079:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232757.019081:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232757.019082:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232757.019085:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5629 00000100:00080000:3.0:1761232757.019086:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232757.019088:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232757.019091:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232757.019097:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607072448/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232757.019121:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232757/real 1761232757] req@0000000090b40c66 x1846780607072256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232782 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232757.019140:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607072256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232782 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232757.019148:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232757.019150:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232757.019153:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607072256:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232757.019172:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072320:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232757.019193:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232757/real 1761232757] req@000000001f8e688e x1846780607072320/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232782 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232757.019200:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607072320/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232782 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232757.019204:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232757.019206:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232757.019208:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072320:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232757.019211:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072384:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232757.019223:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232757/real 1761232757] req@00000000a58ca11e x1846780607072384/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232782 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232757.019229:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607072384/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232782 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232757.019234:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232757.019235:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232757.019236:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072384:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232757.019239:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072448:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232757.019250:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232757/real 1761232757] req@0000000083f16a6b x1846780607072448/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232782 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232757.019256:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607072448/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232782 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232757.019261:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232757.019262:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232757.019263:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072448:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232757.467158:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232758.491165:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232759.515148:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232760.539179:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232761.563103:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232762.139032:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232762.139050:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232762.139071:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232762.139084:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5634 00000100:00080000:0.0:1761232762.139089:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232762.139097:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232762.139146:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607072512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232762.139190:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232762.139195:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232762.139197:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232762.139202:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5634 00000100:00080000:0.0:1761232762.139204:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232762.139208:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232762.139217:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232762.139229:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607072576/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232762.139239:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232762.139243:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232762.139244:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232762.139248:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5634 00000100:00080000:0.0:1761232762.139250:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232762.139253:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232762.139257:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232762.139265:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607072640/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232762.139274:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232762.139278:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232762.139280:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232762.139284:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5634 00000100:00080000:0.0:1761232762.139287:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232762.139291:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232762.139296:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232762.139308:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607072704/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232762.139330:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607072512:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232762.139440:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232762/real 1761232762] req@000000005a096598 x1846780607072512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232787 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232762.139460:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607072512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232787 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232762.139498:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232762.139502:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232762.139507:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607072512:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232762.139515:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072576:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232762.139545:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232762/real 1761232762] req@00000000275aeee5 x1846780607072576/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232787 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232762.139558:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607072576/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232787 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232762.139566:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232762.139570:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232762.139573:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072576:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232762.139579:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072640:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232762.139601:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232762/real 1761232762] req@000000002ef1f54f x1846780607072640/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232787 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232762.139613:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607072640/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232787 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232762.139622:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232762.139624:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232762.139627:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072640:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232762.139658:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072704:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232762.139699:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232762/real 1761232762] req@0000000099514377 x1846780607072704/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232787 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232762.139709:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607072704/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232787 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232762.139717:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232762.139719:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232762.139721:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072704:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232762.587101:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232763.611158:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232764.635056:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232765.659118:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232766.683149:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232767.259087:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232767.259105:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232767.259112:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232767.259128:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5639 00000100:00080000:3.0:1761232767.259134:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232767.259146:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232767.259209:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607072768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232767.259256:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232767.259261:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232767.259263:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232767.259269:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5639 00000100:00080000:3.0:1761232767.259272:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232767.259275:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232767.259288:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232767.259302:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607072832/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232767.259314:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232767.259319:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232767.259321:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232767.259326:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5639 00000100:00080000:3.0:1761232767.259328:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232767.259331:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232767.259337:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232767.259355:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607072768:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232767.259356:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607072896/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232767.259376:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232767.259379:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232767.259381:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232767.259384:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5639 00000100:00080000:3.0:1761232767.259387:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232767.259391:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232767.259396:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232767.259436:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607072960/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232767.259549:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232767/real 1761232767] req@0000000031e043f4 x1846780607072768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232792 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232767.259583:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607072768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232792 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232767.259597:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232767.259601:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232767.259607:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607072768:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232767.259615:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072832:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232767.259644:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232767/real 1761232767] req@000000007361665d x1846780607072832/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232792 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232767.259656:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607072832/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232792 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232767.259664:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232767.259667:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232767.259670:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072832:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232767.259706:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072896:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232767.259732:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232767/real 1761232767] req@00000000f87d0618 x1846780607072896/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232792 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232767.259744:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607072896/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232792 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232767.259754:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232767.259756:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232767.259759:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072896:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232767.259765:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072960:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232767.259786:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232767/real 1761232767] req@00000000c7e4e432 x1846780607072960/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232792 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232767.259798:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607072960/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232792 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232767.259807:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232767.259809:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232767.259811:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607072960:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761232767.707124:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232768.731229:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232769.755101:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232770.779024:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232771.803123:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232772.379062:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232772.379075:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232772.379079:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232772.379090:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5644 00000100:00080000:2.0:1761232772.379094:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232772.379103:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232772.379148:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607073024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232772.379182:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232772.379186:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232772.379187:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232772.379192:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5644 00000100:00080000:2.0:1761232772.379194:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232772.379197:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232772.379206:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232772.379217:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607073088/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232772.379225:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232772.379228:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232772.379230:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232772.379233:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5644 00000100:00080000:2.0:1761232772.379235:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232772.379237:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232772.379242:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232772.379250:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607073152/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232772.379258:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232772.379262:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232772.379263:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232772.379268:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5644 00000100:00080000:2.0:1761232772.379271:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232772.379275:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232772.379280:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232772.379290:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607073216/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232772.379305:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607073024:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232772.379455:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232772/real 1761232772] req@0000000091106a97 x1846780607073024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232797 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232772.379471:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607073024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232797 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232772.379481:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232772.379484:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232772.379487:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607073024:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232772.379493:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073088:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232772.379514:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232772/real 1761232772] req@0000000094135f44 x1846780607073088/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232797 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232772.379523:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607073088/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232797 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232772.379534:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232772.379536:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232772.379538:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073088:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232772.379542:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073152:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232772.379558:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232772/real 1761232772] req@0000000064978143 x1846780607073152/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232797 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232772.379566:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607073152/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232797 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232772.379572:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232772.379574:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232772.379576:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073152:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232772.379618:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073216:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232772.379641:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232772/real 1761232772] req@00000000c069b612 x1846780607073216/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232797 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232772.379651:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607073216/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232797 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232772.379659:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232772.379660:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232772.379663:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073216:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232772.827101:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232773.851023:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232774.875011:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232775.899118:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232776.923045:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232777.498979:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232777.498991:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232777.498996:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232777.499009:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5649 00000100:00080000:2.0:1761232777.499013:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232777.499021:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232777.499066:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607073280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232777.499092:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232777.499095:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232777.499096:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232777.499099:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5649 00000100:00080000:2.0:1761232777.499101:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232777.499103:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232777.499112:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232777.499121:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607073344/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232777.499133:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232777.499136:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232777.499138:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232777.499141:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5649 00000100:00080000:2.0:1761232777.499142:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232777.499144:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232777.499147:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232777.499155:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607073280:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761232777.499157:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607073408/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232777.499183:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232777.499186:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232777.499187:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232777.499191:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5649 00000100:00080000:2.0:1761232777.499192:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232777.499195:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232777.499198:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232777.499209:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607073472/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232777.499279:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232777/real 1761232777] req@00000000f38f27df x1846780607073280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232802 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232777.499313:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607073280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232802 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232777.499325:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232777.499328:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232777.499333:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607073280:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232777.499341:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073344:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232777.499371:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232777/real 1761232777] req@0000000017386078 x1846780607073344/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232802 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232777.499383:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607073344/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232802 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232777.499391:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232777.499394:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232777.499397:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073344:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232777.499423:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073408:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232777.499449:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232777/real 1761232777] req@00000000c069b612 x1846780607073408/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232802 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232777.499460:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607073408/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232802 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232777.499468:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232777.499472:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232777.499475:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073408:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232777.499481:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073472:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232777.499506:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232777/real 1761232777] req@0000000064978143 x1846780607073472/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232802 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232777.499517:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607073472/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232802 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232777.499525:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232777.499528:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232777.499531:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073472:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232777.946987:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232778.971160:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232779.995137:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232781.018995:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232782.042990:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232782.618947:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232782.618959:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232782.618963:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232782.618975:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5655 00000100:00080000:2.0:1761232782.618979:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232782.618987:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232782.619025:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607073536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232782.619058:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232782.619062:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232782.619064:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232782.619068:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5655 00000100:00080000:2.0:1761232782.619070:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232782.619074:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232782.619083:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232782.619103:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607073600/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232782.619112:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232782.619115:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232782.619117:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232782.619120:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5655 00000100:00080000:2.0:1761232782.619123:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232782.619127:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232782.619132:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232782.619144:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607073664/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232782.619153:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607073536:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:2.0:1761232782.619158:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232782.619163:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232782.619165:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232782.619169:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5655 00000100:00080000:2.0:1761232782.619172:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232782.619175:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232782.619180:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232782.619193:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607073728/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232782.619251:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232782/real 1761232782] req@0000000094135f44 x1846780607073536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232807 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232782.619264:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607073536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232807 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232782.619272:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232782.619275:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232782.619278:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607073536:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232782.619283:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073600:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232782.619300:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232782/real 1761232782] req@0000000091106a97 x1846780607073600/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232807 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232782.619312:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607073600/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232807 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232782.619316:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232782.619318:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232782.619319:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073600:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232782.619338:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073664:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232782.619352:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232782/real 1761232782] req@0000000064978143 x1846780607073664/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232807 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232782.619359:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607073664/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232807 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232782.619364:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232782.619365:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232782.619367:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073664:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232782.619370:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073728:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232782.619381:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232782/real 1761232782] req@00000000c069b612 x1846780607073728/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232807 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232782.619386:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607073728/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232807 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232782.619391:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232782.619392:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232782.619395:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073728:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232783.067146:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232784.091128:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232785.115070:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232786.139133:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232787.163118:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232787.739104:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232787.739117:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232787.739122:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232787.739135:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5660 00000100:00080000:0.0:1761232787.739140:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232787.739149:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232787.739255:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607073792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232787.739295:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232787.739300:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232787.739302:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232787.739307:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5660 00000100:00080000:0.0:1761232787.739309:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232787.739315:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232787.739325:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232787.739344:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607073856/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232787.739355:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232787.739359:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232787.739362:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232787.739366:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5660 00000100:00080000:0.0:1761232787.739368:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232787.739371:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232787.739375:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232787.739397:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607073920/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232787.739398:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607073792:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761232787.739435:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232787.739438:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232787.739439:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232787.739444:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5660 00000100:00080000:0.0:1761232787.739447:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232787.739452:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232787.739457:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232787.739471:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607073984/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232787.739561:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073856:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232787.739603:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073920:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232787.739628:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073984:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232788.187189:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232789.211005:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232790.235081:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761232790.811450:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232787/real 1761232790] req@0000000023ab3a3f x1846780607073920/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232812 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232790.811503:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607073920/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232812 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232790.811519:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232790.811524:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232790.811530:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073920:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232790.811539:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232787/real 1761232790] req@0000000001b2f813 x1846780607073984/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232812 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232790.811551:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607073984/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232812 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232790.811560:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232790.811563:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232790.811567:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073984:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232790.811575:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232787/real 1761232790] req@00000000271a9d8e x1846780607073792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232812 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232790.811600:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607073792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232812 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232790.811610:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232790.811613:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232790.811617:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607073792:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232790.811625:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232787/real 1761232790] req@000000000243251f x1846780607073856/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232812 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232790.811638:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607073856/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232812 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232790.811653:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232790.811657:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232790.811660:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607073856:192.168.201.104@tcp:38:kworker/u8:1.0 00010000:00010000:2.0:1761232791.259104:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232792.283208:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232792.858944:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232792.858958:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232792.858963:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232792.858976:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5665 00000100:00080000:0.0:1761232792.858981:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232792.858989:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232792.859034:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607074048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232792.859067:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232792.859072:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232792.859074:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232792.859078:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5665 00000100:00080000:0.0:1761232792.859081:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232792.859084:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232792.859094:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232792.859108:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607074112/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232792.859139:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232792.859143:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232792.859145:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232792.859149:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5665 00000100:00080000:0.0:1761232792.859151:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232792.859155:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232792.859159:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232792.859171:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607074176/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232792.859182:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232792.859189:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232792.859192:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232792.859199:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5665 00000100:00080000:0.0:1761232792.859203:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232792.859210:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232792.859218:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232792.859237:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607074048:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:0.0:1761232792.859240:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607074240/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232792.859380:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232792/real 1761232792] req@00000000b8f2be78 x1846780607074048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232817 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232792.859412:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607074048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232817 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232792.859430:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232792.859436:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232792.859443:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607074048:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232792.859457:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074112:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232792.859513:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232792/real 1761232792] req@00000000fcce404c x1846780607074112/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232817 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232792.859536:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607074112/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232817 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232792.859550:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232792.859554:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232792.859560:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074112:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232792.859571:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074176:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232792.859620:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232792/real 1761232792] req@000000007f60e210 x1846780607074176/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232817 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232792.859643:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607074176/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232817 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232792.859656:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232792.859660:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232792.859665:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074176:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232792.859719:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074240:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232792.859771:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232792/real 1761232792] req@000000000243251f x1846780607074240/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232817 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232792.859792:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607074240/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232817 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232792.859806:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232792.859812:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232792.859817:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074240:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232793.307147:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232794.331209:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232795.355117:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232796.378993:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232797.402989:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232797.979087:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232797.979101:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232797.979105:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232797.979119:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5670 00000100:00080000:1.0:1761232797.979123:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232797.979132:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232797.979200:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607074304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232797.979237:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232797.979242:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232797.979243:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232797.979248:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5670 00000100:00080000:1.0:1761232797.979251:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232797.979254:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232797.979264:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232797.979276:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607074368/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232797.979288:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232797.979293:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232797.979295:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232797.979299:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5670 00000100:00080000:1.0:1761232797.979302:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232797.979307:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232797.979312:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232797.979327:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607074304:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232797.979328:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607074432/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232797.979350:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232797.979354:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232797.979355:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232797.979359:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5670 00000100:00080000:1.0:1761232797.979361:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232797.979365:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232797.979370:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232797.979383:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607074496/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232797.979452:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232797/real 1761232797] req@00000000db10af3c x1846780607074304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232822 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232797.979474:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607074304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232822 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232797.979486:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232797.979490:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232797.979495:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607074304:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232797.979503:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074368:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232797.979529:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232797/real 1761232797] req@00000000659d78e3 x1846780607074368/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232822 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232797.979542:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607074368/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232822 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232797.979550:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232797.979552:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232797.979555:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074368:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232797.979585:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074432:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232797.979613:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232797/real 1761232797] req@00000000b72feabb x1846780607074432/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232822 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232797.979625:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607074432/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232822 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232797.979634:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232797.979636:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232797.979639:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074432:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232797.979644:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074496:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232797.979687:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232797/real 1761232797] req@00000000d07717c4 x1846780607074496/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232822 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232797.979700:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607074496/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232822 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232797.979709:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232797.979712:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232797.979715:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074496:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761232798.427085:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232799.450992:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232800.475178:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232801.499102:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761232802.523062:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232803.099129:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232803.099143:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232803.099147:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232803.099160:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5675 00000100:00080000:0.0:1761232803.099164:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232803.099173:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232803.099218:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607074560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232803.099251:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232803.099256:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232803.099259:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232803.099264:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5675 00000100:00080000:0.0:1761232803.099266:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232803.099270:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232803.099280:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232803.099294:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607074624/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232803.099304:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232803.099308:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232803.099310:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232803.099314:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5675 00000100:00080000:0.0:1761232803.099315:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232803.099318:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232803.099322:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232803.099331:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607074688/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232803.099340:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232803.099343:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232803.099344:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232803.099347:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5675 00000100:00080000:0.0:1761232803.099349:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232803.099353:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232803.099357:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232803.099365:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607074752/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232803.099506:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607074560:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232803.099607:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232803/real 1761232803] req@00000000271a9d8e x1846780607074560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232828 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232803.099626:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607074560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232828 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232803.099645:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232803.099648:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232803.099652:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607074560:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232803.099660:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074624:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232803.099686:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232803/real 1761232803] req@0000000001b2f813 x1846780607074624/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232828 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232803.099698:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607074624/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232828 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232803.099706:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232803.099708:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232803.099711:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074624:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232803.099716:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074688:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232803.099739:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232803/real 1761232803] req@0000000023ab3a3f x1846780607074688/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232828 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232803.099751:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607074688/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232828 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232803.099760:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232803.099762:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232803.099765:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074688:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232803.099770:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074752:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232803.099791:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232803/real 1761232803] req@0000000099514377 x1846780607074752/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232828 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232803.099802:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607074752/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232828 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232803.099811:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232803.099813:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232803.099816:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074752:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232803.547129:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232804.571154:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232805.595111:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232806.619141:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761232807.643066:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232808.219071:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232808.219083:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232808.219087:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232808.219097:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5680 00000100:00080000:3.0:1761232808.219101:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232808.219107:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232808.219143:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607074816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232808.219179:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232808.219182:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232808.219184:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232808.219188:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5680 00000100:00080000:3.0:1761232808.219190:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232808.219194:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232808.219202:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232808.219215:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607074880/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232808.219224:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232808.219227:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232808.219229:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232808.219232:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5680 00000100:00080000:3.0:1761232808.219235:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232808.219238:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232808.219242:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232808.219253:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607074944/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232808.219263:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232808.219267:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232808.219269:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232808.219272:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607074816:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761232808.219277:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5680 00000100:00080000:3.0:1761232808.219280:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232808.219283:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232808.219287:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232808.219300:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607075008/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232808.219353:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232808/real 1761232808] req@0000000086ccb527 x1846780607074816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232833 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232808.219365:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607074816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232833 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232808.219372:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232808.219374:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232808.219377:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607074816:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232808.219381:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074880:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232808.219397:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232808/real 1761232808] req@00000000bd684ccc x1846780607074880/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232833 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232808.219405:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607074880/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232833 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232808.219409:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232808.219410:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232808.219412:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074880:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232808.219430:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074944:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232808.219443:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232808/real 1761232808] req@0000000018a8f5da x1846780607074944/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232833 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232808.219449:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607074944/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232833 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232808.219455:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232808.219456:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232808.219457:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607074944:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232808.219460:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075008:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232808.219470:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232808/real 1761232808] req@000000005a9bdefd x1846780607075008/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232833 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232808.219476:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607075008/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232833 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232808.219481:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232808.219482:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232808.219485:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075008:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232808.667062:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232809.691166:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232810.715159:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232811.739206:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761232812.763102:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232813.339052:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232813.339062:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232813.339065:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232813.339075:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5685 00000100:00080000:1.0:1761232813.339078:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232813.339084:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232813.339119:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607075072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232813.339144:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232813.339147:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232813.339148:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232813.339151:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5685 00000100:00080000:1.0:1761232813.339153:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232813.339156:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232813.339164:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232813.339173:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607075136/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232813.339181:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761232813.339181:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607075072:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761232813.339189:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232813.339191:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232813.339195:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5685 00000100:00080000:1.0:1761232813.339196:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232813.339198:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232813.339203:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232813.339213:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607075200/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232813.339219:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232813.339222:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232813.339223:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232813.339225:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5685 00000100:00080000:1.0:1761232813.339226:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232813.339229:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232813.339231:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232813.339237:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607075264/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232813.339270:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232813/real 1761232813] req@00000000d07717c4 x1846780607075072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232838 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232813.339289:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607075072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232838 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232813.339296:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232813.339299:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232813.339302:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607075072:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232813.339321:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075136:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232813.339339:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232813/real 1761232813] req@00000000b72feabb x1846780607075136/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232838 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232813.339346:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607075136/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232838 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232813.339351:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232813.339353:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232813.339355:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075136:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232813.339358:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075200:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232813.339371:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232813/real 1761232813] req@00000000659d78e3 x1846780607075200/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232838 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232813.339378:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607075200/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232838 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232813.339400:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232813.339402:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232813.339404:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075200:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232813.339409:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075264:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232813.339426:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232813/real 1761232813] req@00000000db10af3c x1846780607075264/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232838 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232813.339435:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607075264/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232838 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232813.339441:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232813.339442:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232813.339444:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075264:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232813.787180:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232814.811144:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232815.835150:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232816.859153:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761232817.882991:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232818.459146:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232818.459157:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232818.459161:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232818.459173:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5690 00000100:00080000:0.0:1761232818.459176:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232818.459185:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232818.459223:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607075328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232818.459249:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232818.459253:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232818.459255:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232818.459259:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5690 00000100:00080000:0.0:1761232818.459261:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232818.459264:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232818.459271:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232818.459283:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607075392/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232818.459291:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232818.459294:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232818.459295:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232818.459300:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5690 00000100:00080000:0.0:1761232818.459303:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232818.459305:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232818.459309:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232818.459319:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607075456/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232818.459328:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232818.459330:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232818.459333:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232818.459337:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5690 00000100:00080000:0.0:1761232818.459338:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232818.459341:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232818.459345:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232818.459357:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607075520/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232818.459380:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607075328:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232818.459536:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232818/real 1761232818] req@000000002ef1f54f x1846780607075328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232843 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232818.459572:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607075328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232843 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232818.459587:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232818.459592:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232818.459597:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607075328:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232818.459606:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075392:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232818.459637:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232818/real 1761232818] req@00000000275aeee5 x1846780607075392/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232843 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232818.459650:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607075392/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232843 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232818.459660:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232818.459662:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232818.459665:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075392:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232818.459671:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075456:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232818.459694:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232818/real 1761232818] req@000000005a096598 x1846780607075456/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232843 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232818.459705:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607075456/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232843 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232818.459714:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232818.459716:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232818.459719:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075456:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232818.459758:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075520:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232818.459782:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232818/real 1761232818] req@0000000099514377 x1846780607075520/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232843 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232818.459809:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607075520/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232843 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232818.459817:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232818.459819:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232818.459909:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075520:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232818.907148:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232819.931064:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232820.955011:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232821.979063:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761232823.003018:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232823.578971:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232823.578990:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232823.578996:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232823.579013:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5695 00000100:00080000:3.0:1761232823.579019:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232823.579029:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232823.579104:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607075584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232823.579141:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232823.579148:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232823.579163:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232823.579172:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5695 00000100:00080000:3.0:1761232823.579181:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232823.579188:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232823.579202:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232823.579220:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607075584:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232823.579222:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607075648/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232823.579240:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232823.579245:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232823.579248:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232823.579254:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5695 00000100:00080000:3.0:1761232823.579257:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232823.579262:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232823.579269:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232823.579284:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607075712/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232823.579298:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232823.579302:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232823.579305:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232823.579309:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5695 00000100:00080000:3.0:1761232823.579313:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232823.579318:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232823.579324:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232823.579338:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607075776/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232823.579341:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232823/real 1761232823] req@00000000daa39bb1 x1846780607075584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232848 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232823.579369:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607075584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232848 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232823.579386:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232823.579392:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232823.579398:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607075584:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232823.579436:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075648:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232823.579481:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232823/real 1761232823] req@000000003ad149cc x1846780607075648/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232848 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232823.579503:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607075648/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232848 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232823.579513:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232823.579515:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232823.579519:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075648:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232823.579526:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075712:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232823.579549:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232823/real 1761232823] req@0000000088473b3e x1846780607075712/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232848 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232823.579561:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607075712/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232848 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232823.579570:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232823.579572:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232823.579575:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075712:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232823.579580:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075776:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232823.579599:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232823/real 1761232823] req@000000004e3f612a x1846780607075776/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232848 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232823.579610:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607075776/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232848 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232823.579618:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232823.579620:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232823.579624:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075776:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232824.027151:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232825.051055:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232826.075104:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232827.099162:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761232828.123063:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232828.699087:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232828.699101:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232828.699107:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232828.699119:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5701 00000100:00080000:1.0:1761232828.699124:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232828.699132:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232828.699183:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607075840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232828.699227:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232828.699231:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232828.699233:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232828.699239:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5701 00000100:00080000:1.0:1761232828.699241:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232828.699245:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232828.699256:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232828.699270:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607075904/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232828.699281:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232828.699286:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232828.699288:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232828.699292:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5701 00000100:00080000:1.0:1761232828.699295:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232828.699298:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232828.699306:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232828.699310:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607075840:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232828.699322:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607075968/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232828.699334:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232828.699338:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232828.699341:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232828.699344:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5701 00000100:00080000:1.0:1761232828.699347:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232828.699350:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232828.699355:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232828.699368:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607076032/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232828.699428:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232828/real 1761232828] req@00000000db10af3c x1846780607075840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232853 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232828.699450:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607075840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232853 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232828.699462:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232828.699466:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232828.699471:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607075840:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232828.699479:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075904:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232828.699506:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232828/real 1761232828] req@00000000659d78e3 x1846780607075904/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232853 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232828.699528:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607075904/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232853 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232828.699536:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232828.699538:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232828.699541:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075904:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232828.699570:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075968:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232828.699594:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232828/real 1761232828] req@00000000b72feabb x1846780607075968/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232853 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232828.699605:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607075968/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232853 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232828.699615:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232828.699617:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232828.699620:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607075968:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232828.699626:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076032:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232828.699644:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232828/real 1761232828] req@00000000d07717c4 x1846780607076032/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232853 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232828.699656:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607076032/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232853 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232828.699665:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232828.699667:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232828.699671:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076032:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232829.147174:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232830.171171:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232831.195010:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232832.219034:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232833.243032:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232833.818944:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232833.818953:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232833.818957:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232833.818965:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5706 00000100:00080000:0.0:1761232833.818968:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232833.818973:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232833.819005:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607076096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232833.819036:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232833.819038:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232833.819040:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232833.819042:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5706 00000100:00080000:0.0:1761232833.819044:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232833.819046:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232833.819052:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232833.819059:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607076160/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232833.819064:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232833.819067:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232833.819068:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232833.819071:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5706 00000100:00080000:0.0:1761232833.819072:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232833.819074:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232833.819076:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232833.819081:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607076224/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232833.819086:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232833.819088:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232833.819089:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232833.819092:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5706 00000100:00080000:0.0:1761232833.819093:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232833.819095:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232833.819098:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232833.819103:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607076288/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232833.819115:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607076096:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232833.819201:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232833/real 1761232833] req@0000000023ab3a3f x1846780607076096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232858 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232833.819214:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607076096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232858 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232833.819222:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232833.819224:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232833.819226:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607076096:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232833.819230:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076160:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232833.819244:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232833/real 1761232833] req@0000000001b2f813 x1846780607076160/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232858 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232833.819250:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607076160/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232858 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232833.819255:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232833.819256:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232833.819257:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076160:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232833.819260:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076224:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232833.819273:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232833/real 1761232833] req@00000000271a9d8e x1846780607076224/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232858 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232833.819279:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607076224/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232858 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232833.819285:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232833.819286:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232833.819288:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076224:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232833.819306:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076288:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232833.819318:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232833/real 1761232833] req@000000000243251f x1846780607076288/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232858 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232833.819324:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607076288/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232858 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232833.819329:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232833.819330:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232833.819332:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076288:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232834.266983:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232835.291100:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232836.315057:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232837.339137:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232838.363119:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232838.938948:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232838.938977:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232838.938981:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232838.938993:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5711 00000100:00080000:0.0:1761232838.938998:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232838.939006:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232838.939050:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607076352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232838.939089:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232838.939094:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232838.939095:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232838.939100:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5711 00000100:00080000:0.0:1761232838.939102:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232838.939105:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232838.939114:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232838.939128:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607076416/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232838.939137:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232838.939141:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232838.939142:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232838.939146:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5711 00000100:00080000:0.0:1761232838.939148:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232838.939152:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232838.939156:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232838.939165:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607076480/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232838.939175:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232838.939179:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232838.939180:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232838.939184:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5711 00000100:00080000:0.0:1761232838.939204:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232838.939208:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232838.939212:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232838.939222:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607076352:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:0.0:1761232838.939227:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607076544/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232838.939323:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232838/real 1761232838] req@000000007f60e210 x1846780607076352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232863 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232838.939341:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607076352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232863 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232838.939352:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232838.939356:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232838.939360:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607076352:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232838.939367:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076416:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232838.939392:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232838/real 1761232838] req@00000000fcce404c x1846780607076416/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232863 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232838.939403:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607076416/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232863 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232838.939410:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232838.939412:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232838.939415:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076416:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232838.939420:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076480:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232838.939438:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232838/real 1761232838] req@00000000b8f2be78 x1846780607076480/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232863 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232838.939448:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607076480/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232863 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232838.939456:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232838.939458:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232838.939460:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076480:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232838.939512:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076544:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232838.939529:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232838/real 1761232838] req@000000000243251f x1846780607076544/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232863 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232838.939536:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607076544/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232863 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232838.939542:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232838.939545:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232838.939547:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076544:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232839.387123:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232840.411065:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232841.435083:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232842.459152:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232843.483115:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232844.059103:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232844.059122:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232844.059128:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232844.059145:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5716 00000100:00080000:3.0:1761232844.059152:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232844.059164:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232844.059222:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607076608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232844.059277:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232844.059284:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232844.059287:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232844.059295:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5716 00000100:00080000:3.0:1761232844.059299:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232844.059306:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232844.059320:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232844.059346:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607076672/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232844.059368:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232844.059375:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232844.059378:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232844.059382:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607076608:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761232844.059392:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5716 00000100:00080000:3.0:1761232844.059399:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232844.059407:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232844.059416:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232844.059442:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607076736/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232844.059459:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232844.059467:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232844.059470:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232844.059476:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5716 00000100:00080000:3.0:1761232844.059480:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232844.059487:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232844.059496:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232844.059510:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232844/real 1761232844] req@000000004e3f612a x1846780607076608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232869 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232844.059522:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607076800/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232844.059581:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607076608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232869 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232844.059596:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232844.059600:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232844.059606:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607076608:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232844.059642:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076672:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232844.059679:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232844/real 1761232844] req@0000000088473b3e x1846780607076672/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232869 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232844.059701:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607076672/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232869 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232844.059710:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232844.059712:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232844.059716:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076672:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232844.059722:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076736:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232844.059745:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232844/real 1761232844] req@000000003ad149cc x1846780607076736/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232869 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232844.059756:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607076736/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232869 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232844.059765:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232844.059767:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232844.059770:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076736:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232844.059776:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076800:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232844.059796:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232844/real 1761232844] req@00000000daa39bb1 x1846780607076800/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232869 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232844.059807:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607076800/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232869 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232844.059816:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232844.059818:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232844.059822:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076800:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232844.507014:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232845.531106:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232846.555190:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232847.579172:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232848.603071:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232849.178945:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232849.178959:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232849.178964:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232849.178976:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5721 00000100:00080000:3.0:1761232849.178980:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232849.178988:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232849.179029:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607076864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232849.179063:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232849.179068:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232849.179069:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232849.179091:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5721 00000100:00080000:3.0:1761232849.179095:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232849.179102:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232849.179117:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232849.179141:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607076928/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232849.179158:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232849.179163:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232849.179177:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232849.179184:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5721 00000100:00080000:3.0:1761232849.179189:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232849.179196:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232849.179205:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232849.179228:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607076864:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232849.179231:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607076992/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232849.179262:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232849.179268:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232849.179272:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232849.179280:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5721 00000100:00080000:3.0:1761232849.179284:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232849.179290:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232849.179300:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232849.179322:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607077056/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232849.179361:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232849/real 1761232849] req@000000005a9bdefd x1846780607076864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232874 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232849.179403:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607076864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232874 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232849.179416:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232849.179420:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232849.179425:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607076864:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232849.179433:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076928:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232849.179460:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232849/real 1761232849] req@0000000018a8f5da x1846780607076928/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232874 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232849.179542:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607076928/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232874 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232849.179550:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232849.179552:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232849.179555:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076928:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232849.179589:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076992:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232849.179618:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232849/real 1761232849] req@00000000bd684ccc x1846780607076992/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232874 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232849.179631:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607076992/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232874 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232849.179643:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232849.179646:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232849.179650:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607076992:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232849.179657:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077056:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232849.179686:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232849/real 1761232849] req@0000000086ccb527 x1846780607077056/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232874 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232849.179695:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607077056/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232874 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232849.179702:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232849.179704:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232849.179706:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077056:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232849.627088:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232850.651134:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232851.675145:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232852.699199:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232853.723223:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232854.298975:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232854.298986:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232854.298990:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232854.298999:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5726 00000100:00080000:3.0:1761232854.299002:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232854.299008:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232854.299045:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607077120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232854.299072:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232854.299075:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232854.299076:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232854.299079:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5726 00000100:00080000:3.0:1761232854.299082:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232854.299084:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232854.299091:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232854.299099:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607077184/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232854.299106:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232854.299109:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232854.299110:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232854.299112:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5726 00000100:00080000:3.0:1761232854.299114:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232854.299116:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232854.299121:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232854.299129:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607077248/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232854.299135:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232854.299138:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232854.299139:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232854.299142:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5726 00000100:00080000:3.0:1761232854.299143:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232854.299146:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232854.299149:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232854.299155:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607077312/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232854.299174:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607077120:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232854.299258:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077184:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232854.299276:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077248:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232854.299298:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077312:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232854.747125:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232855.771047:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232856.795189:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761232857.371424:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232854/real 1761232857] req@0000000031e043f4 x1846780607077312/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232879 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232857.371470:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607077312/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232879 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232857.371501:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232857.371507:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232857.371515:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077312:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232857.371526:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232854/real 1761232857] req@00000000c7e4e432 x1846780607077120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232879 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232857.371547:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607077120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232879 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232857.371559:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232857.371563:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232857.371569:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607077120:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232857.371578:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232854/real 1761232857] req@00000000f87d0618 x1846780607077184/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232879 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232857.371596:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607077184/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232879 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232857.371610:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232857.371614:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232857.371619:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077184:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232857.371628:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232854/real 1761232857] req@000000007361665d x1846780607077248/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232879 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232857.371662:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607077248/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232879 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232857.371691:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232857.371695:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232857.371701:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077248:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232857.819030:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232858.843161:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232859.419069:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232859.419084:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232859.419089:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232859.419101:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5731 00000100:00080000:1.0:1761232859.419106:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232859.419113:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232859.419155:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607077376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232859.419197:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232859.419202:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232859.419204:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232859.419209:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5731 00000100:00080000:1.0:1761232859.419211:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232859.419215:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232859.419225:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232859.419237:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607077440/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232859.419248:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232859.419252:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232859.419253:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232859.419258:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5731 00000100:00080000:1.0:1761232859.419261:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232859.419264:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232859.419269:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232859.419283:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607077504/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232859.419298:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232859.419304:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232859.419308:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232859.419313:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5731 00000100:00080000:1.0:1761232859.419317:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232859.419323:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232859.419330:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232859.419351:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607077568/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232859.419352:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607077376:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232859.419551:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232859/real 1761232859] req@00000000d07717c4 x1846780607077376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232884 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232859.419582:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607077376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232884 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232859.419600:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232859.419606:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232859.419612:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607077376:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232859.419626:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077440:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232859.419684:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232859/real 1761232859] req@00000000b72feabb x1846780607077440/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232884 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232859.419717:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607077440/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232884 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232859.419730:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232859.419735:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232859.419741:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077440:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232859.419752:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077504:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232859.419803:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232859/real 1761232859] req@00000000659d78e3 x1846780607077504/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232884 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232859.419825:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607077504/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232884 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232859.419886:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232859.419891:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232859.419907:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077504:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232859.419954:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077568:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232859.420008:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232859/real 1761232859] req@00000000db10af3c x1846780607077568/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232884 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232859.420027:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607077568/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232884 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232859.420041:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232859.420044:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232859.420050:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077568:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232859.867098:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232860.891141:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232861.914998:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232862.938991:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232863.963073:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232864.539059:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232864.539073:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232864.539078:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232864.539092:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5736 00000100:00080000:3.0:1761232864.539096:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232864.539105:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232864.539149:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607077632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232864.539188:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232864.539192:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232864.539194:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232864.539199:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5736 00000100:00080000:3.0:1761232864.539202:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232864.539205:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232864.539216:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232864.539231:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607077696/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232864.539243:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232864.539248:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232864.539250:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232864.539256:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607077632:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761232864.539260:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5736 00000100:00080000:3.0:1761232864.539270:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232864.539275:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232864.539282:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232864.539327:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607077760/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232864.539339:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232864.539343:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232864.539346:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232864.539350:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5736 00000100:00080000:3.0:1761232864.539353:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232864.539356:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232864.539361:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232864.539373:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607077824/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232864.539426:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232864/real 1761232864] req@0000000083f16a6b x1846780607077632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232889 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232864.539449:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607077632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232889 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232864.539461:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232864.539465:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232864.539469:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607077632:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232864.539503:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077696:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232864.539533:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232864/real 1761232864] req@00000000a58ca11e x1846780607077696/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232889 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232864.539545:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607077696/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232889 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232864.539554:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232864.539557:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232864.539560:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077696:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232864.539566:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077760:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232864.539588:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232864/real 1761232864] req@000000001f8e688e x1846780607077760/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232889 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232864.539599:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607077760/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232889 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232864.539609:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232864.539611:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232864.539614:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077760:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232864.539619:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077824:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232864.539639:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232864/real 1761232864] req@0000000090b40c66 x1846780607077824/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232889 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232864.539650:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607077824/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232889 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232864.539660:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232864.539662:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232864.539664:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077824:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232864.987094:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232866.011087:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232867.035117:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232868.059122:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232869.083193:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232869.659069:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232869.659082:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232869.659087:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232869.659099:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5742 00000100:00080000:0.0:1761232869.659104:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232869.659112:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232869.659156:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607077888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232869.659193:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232869.659198:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232869.659200:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232869.659204:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5742 00000100:00080000:0.0:1761232869.659207:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232869.659210:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232869.659220:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232869.659232:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607077952/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232869.659286:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232869.659291:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232869.659293:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232869.659297:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5742 00000100:00080000:0.0:1761232869.659300:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232869.659304:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232869.659311:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232869.659323:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607078016/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232869.659334:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232869.659339:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232869.659341:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232869.659345:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5742 00000100:00080000:0.0:1761232869.659347:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232869.659350:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232869.659355:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232869.659358:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607077888:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761232869.659370:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607078080/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232869.659475:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232869/real 1761232869] req@00000000271a9d8e x1846780607077888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232894 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232869.659497:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607077888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232894 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232869.659509:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232869.659513:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232869.659518:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607077888:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232869.659526:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077952:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232869.659553:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232869/real 1761232869] req@0000000001b2f813 x1846780607077952/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232894 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232869.659565:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607077952/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232894 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232869.659574:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232869.659576:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232869.659579:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607077952:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232869.659585:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078016:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232869.659607:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232869/real 1761232869] req@0000000023ab3a3f x1846780607078016/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232894 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232869.659619:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607078016/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232894 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232869.659628:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232869.659630:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232869.659634:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078016:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232869.659676:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078080:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232869.659727:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232869/real 1761232869] req@0000000099514377 x1846780607078080/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232894 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232869.659749:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607078080/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232894 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232869.659764:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232869.659767:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232869.659771:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078080:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232870.107168:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232871.131100:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232872.155134:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232873.179137:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232874.203040:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232874.778976:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232874.778990:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232874.778995:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232874.779009:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5747 00000100:00080000:0.0:1761232874.779013:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232874.779022:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232874.779074:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607078144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232874.779109:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232874.779114:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232874.779116:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232874.779120:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5747 00000100:00080000:0.0:1761232874.779124:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232874.779128:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232874.779138:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232874.779151:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607078208/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232874.779161:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232874.779165:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232874.779167:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232874.779170:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5747 00000100:00080000:0.0:1761232874.779172:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232874.779175:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232874.779179:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232874.779213:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607078272/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232874.779224:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232874.779228:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232874.779230:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232874.779234:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5747 00000100:00080000:0.0:1761232874.779237:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232874.779241:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232874.779246:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232874.779258:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607078336/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232874.779268:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607078144:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232874.779380:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232874/real 1761232874] req@000000005a096598 x1846780607078144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232899 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232874.779401:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607078144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232899 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232874.779413:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232874.779418:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232874.779422:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607078144:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232874.779431:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078208:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232874.779457:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232874/real 1761232874] req@00000000275aeee5 x1846780607078208/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232899 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232874.779470:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607078208/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232899 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232874.779478:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232874.779481:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232874.779483:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078208:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232874.779489:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078272:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232874.779510:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232874/real 1761232874] req@000000002ef1f54f x1846780607078272/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232899 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232874.779521:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607078272/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232899 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232874.779531:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232874.779533:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232874.779535:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078272:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232874.779566:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078336:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232874.779589:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232874/real 1761232874] req@0000000099514377 x1846780607078336/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232899 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232874.779609:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607078336/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232899 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232874.779619:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232874.779621:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232874.779623:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078336:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232875.227031:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232876.251003:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232877.275040:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232878.299139:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232879.323059:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232879.898952:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232879.898961:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232879.898964:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232879.898973:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5752 00000100:00080000:0.0:1761232879.898976:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232879.898981:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232879.899012:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607078400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232879.899034:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232879.899037:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232879.899038:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232879.899040:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5752 00000100:00080000:0.0:1761232879.899041:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232879.899043:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232879.899050:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232879.899056:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607078464/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232879.899061:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232879.899063:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232879.899064:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232879.899067:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5752 00000100:00080000:0.0:1761232879.899068:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232879.899071:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232879.899073:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232879.899080:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607078528/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232879.899085:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232879.899087:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232879.899088:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232879.899090:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5752 00000100:00080000:0.0:1761232879.899091:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232879.899093:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232879.899095:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232879.899101:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607078592/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232879.899121:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607078400:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232879.899225:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232879/real 1761232879] req@0000000023ab3a3f x1846780607078400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232904 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232879.899242:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607078400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232904 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232879.899249:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232879.899251:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232879.899254:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607078400:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232879.899258:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078464:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232879.899275:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232879/real 1761232879] req@0000000001b2f813 x1846780607078464/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232904 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232879.899282:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607078464/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232904 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232879.899286:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232879.899287:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232879.899289:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078464:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232879.899295:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078528:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232879.899319:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232879/real 1761232879] req@00000000271a9d8e x1846780607078528/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232904 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232879.899330:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607078528/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232904 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232879.899337:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232879.899339:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232879.899343:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078528:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232879.899348:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078592:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232879.899371:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232879/real 1761232879] req@000000000243251f x1846780607078592/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232904 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232879.899381:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607078592/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232904 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232879.899389:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232879.899391:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232879.899393:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078592:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232880.347064:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232881.371012:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232882.395044:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232883.419114:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232884.443107:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232885.019128:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232885.019145:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232885.019152:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232885.019168:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5757 00000100:00080000:1.0:1761232885.019173:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232885.019185:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232885.019244:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607078656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232885.019287:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232885.019292:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232885.019294:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232885.019299:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5757 00000100:00080000:1.0:1761232885.019301:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232885.019305:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232885.019315:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232885.019328:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607078720/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232885.019337:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232885.019341:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232885.019342:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232885.019346:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5757 00000100:00080000:1.0:1761232885.019348:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232885.019350:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232885.019355:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232885.019363:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607078784/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232885.019371:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232885.019376:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232885.019379:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232885.019387:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5757 00000100:00080000:1.0:1761232885.019391:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232885.019398:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232885.019405:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232885.019421:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607078848/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232885.019431:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607078656:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232885.019567:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232885/real 1761232885] req@00000000db10af3c x1846780607078656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232910 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232885.019589:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607078656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232910 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232885.019602:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232885.019606:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232885.019611:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607078656:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232885.019618:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078720:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232885.019644:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232885/real 1761232885] req@00000000659d78e3 x1846780607078720/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232910 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232885.019665:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607078720/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232910 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232885.019673:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232885.019675:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232885.019678:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078720:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232885.019683:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078784:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232885.019705:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232885/real 1761232885] req@00000000b72feabb x1846780607078784/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232910 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232885.019716:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607078784/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232910 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232885.019724:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232885.019726:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232885.019729:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078784:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232885.019763:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078848:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232885.019784:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232885/real 1761232885] req@00000000d07717c4 x1846780607078848/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232910 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232885.019795:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607078848/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232910 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232885.019803:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232885.019806:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232885.019809:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078848:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232885.467240:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232886.491166:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232887.515111:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232888.539069:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232889.563147:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232890.139872:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232890.139881:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232890.139885:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232890.139893:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5762 00000100:00080000:1.0:1761232890.139896:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232890.139901:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232890.139932:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607078912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232890.139950:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232890.139953:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232890.139954:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232890.139957:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5762 00000100:00080000:1.0:1761232890.139958:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232890.139960:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232890.139966:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232890.139973:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607078976/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232890.139978:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232890.139981:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232890.139982:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232890.139984:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5762 00000100:00080000:1.0:1761232890.139986:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232890.139987:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232890.139989:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232890.139994:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607079040/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232890.139999:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232890.140001:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232890.140002:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232890.140004:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5762 00000100:00080000:1.0:1761232890.140005:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232890.140020:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232890.140023:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232890.140031:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607079104/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232890.140042:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607078912:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232890.140117:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232890/real 1761232890] req@00000000d07717c4 x1846780607078912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232915 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232890.140134:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607078912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232915 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232890.140145:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232890.140149:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232890.140153:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607078912:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232890.140160:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078976:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232890.140188:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232890/real 1761232890] req@00000000b72feabb x1846780607078976/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232915 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232890.140197:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607078976/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232915 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232890.140202:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232890.140204:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232890.140206:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607078976:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:3.0:1761232890.140209:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079040:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232890.140226:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232890/real 1761232890] req@00000000659d78e3 x1846780607079040/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232915 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232890.140232:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607079040/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232915 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232890.140238:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232890.140239:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232890.140241:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079040:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232890.140259:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079104:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761232890.140271:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232890/real 1761232890] req@00000000db10af3c x1846780607079104/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232915 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761232890.140277:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607079104/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232915 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761232890.140282:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232890.140283:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232890.140284:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079104:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232890.587139:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232891.611194:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232892.635091:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232893.659060:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232894.683086:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232895.259009:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232895.259018:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232895.259021:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232895.259030:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5767 00000100:00080000:3.0:1761232895.259033:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232895.259038:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232895.259070:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607079168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232895.259093:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232895.259095:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232895.259096:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232895.259100:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5767 00000100:00080000:3.0:1761232895.259101:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232895.259104:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232895.259111:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232895.259118:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607079232/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232895.259125:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232895.259127:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232895.259129:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232895.259130:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607079168:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761232895.259135:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5767 00000100:00080000:3.0:1761232895.259138:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232895.259140:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232895.259143:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232895.259151:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607079296/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232895.259155:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232895.259157:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232895.259159:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232895.259161:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5767 00000100:00080000:3.0:1761232895.259163:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232895.259165:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232895.259167:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232895.259173:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607079360/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232895.259211:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232895/real 1761232895] req@000000006572b591 x1846780607079168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232920 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232895.259223:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607079168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232920 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232895.259230:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232895.259233:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232895.259235:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607079168:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232895.259253:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079232:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232895.259269:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232895/real 1761232895] req@00000000fc01c41c x1846780607079232/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232920 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232895.259276:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607079232/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232920 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232895.259280:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232895.259281:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232895.259283:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079232:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232895.259286:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079296:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232895.259297:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232895/real 1761232895] req@00000000408f088c x1846780607079296/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232920 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232895.259303:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607079296/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232920 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232895.259308:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232895.259310:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232895.259311:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079296:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232895.259314:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079360:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232895.259324:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232895/real 1761232895] req@000000002ce934d3 x1846780607079360/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232920 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232895.259334:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607079360/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232920 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232895.259338:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232895.259339:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232895.259341:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079360:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232895.707051:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232896.731045:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232897.755065:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232898.779031:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232899.803004:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232900.378981:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232900.378992:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232900.378995:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232900.379006:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5772 00000100:00080000:1.0:1761232900.379009:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232900.379015:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232900.379050:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607079424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232900.379070:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232900.379073:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232900.379074:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232900.379077:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5772 00000100:00080000:1.0:1761232900.379078:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232900.379080:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232900.379086:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232900.379094:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607079488/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232900.379100:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232900.379104:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232900.379105:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232900.379107:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5772 00000100:00080000:1.0:1761232900.379109:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232900.379110:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232900.379113:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232900.379116:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607079424:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232900.379122:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607079552/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232900.379129:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232900.379131:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232900.379132:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232900.379133:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5772 00000100:00080000:1.0:1761232900.379135:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232900.379137:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232900.379140:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232900.379147:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607079616/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761232900.379207:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232900/real 1761232900] req@00000000db10af3c x1846780607079424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232925 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232900.379225:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607079424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232925 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232900.379235:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232900.379238:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232900.379242:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607079424:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232900.379249:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079488:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232900.379278:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232900/real 1761232900] req@00000000659d78e3 x1846780607079488/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232925 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232900.379289:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607079488/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232925 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232900.379296:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232900.379298:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232900.379301:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079488:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:3.0:1761232900.379325:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079552:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232900.379352:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232900/real 1761232900] req@00000000b72feabb x1846780607079552/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232925 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232900.379363:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607079552/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232925 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232900.379371:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232900.379374:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232900.379377:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079552:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232900.379383:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079616:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761232900.379405:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232900/real 1761232900] req@00000000d07717c4 x1846780607079616/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232925 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232900.379417:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607079616/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232925 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:3.0:1761232900.379424:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761232900.379426:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:3.0:1761232900.379429:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079616:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232900.827033:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232901.851007:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232902.875019:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232903.898990:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232904.922989:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232905.498942:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232905.498954:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232905.498958:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232905.498969:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5777 00000100:00080000:3.0:1761232905.498973:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232905.498980:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232905.499016:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607079680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232905.499045:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232905.499050:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232905.499051:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232905.499056:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5777 00000100:00080000:3.0:1761232905.499059:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232905.499063:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232905.499073:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232905.499083:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607079680:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232905.499089:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607079744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232905.499102:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232905.499106:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232905.499107:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232905.499111:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5777 00000100:00080000:3.0:1761232905.499114:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232905.499117:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232905.499123:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232905.499136:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607079808/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232905.499145:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232905.499149:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232905.499151:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232905.499155:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5777 00000100:00080000:3.0:1761232905.499157:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232905.499161:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232905.499166:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232905.499167:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232905/real 1761232905] req@000000002ce934d3 x1846780607079680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232930 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:3.0:1761232905.499181:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607079872/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232905.499188:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607079680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232930 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232905.499222:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232905.499227:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232905.499232:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607079680:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232905.499257:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079744:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232905.499290:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232905/real 1761232905] req@00000000408f088c x1846780607079744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232930 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232905.499302:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607079744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232930 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232905.499309:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232905.499311:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232905.499314:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079744:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232905.499320:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079808:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232905.499345:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232905/real 1761232905] req@00000000fc01c41c x1846780607079808/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232930 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232905.499355:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607079808/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232930 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232905.499363:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232905.499365:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232905.499367:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079808:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232905.499372:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079872:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232905.499395:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232905/real 1761232905] req@000000006572b591 x1846780607079872/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232930 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232905.499412:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607079872/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232930 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232905.499419:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232905.499421:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232905.499423:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607079872:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232905.947001:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232906.971005:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232907.995036:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232909.019051:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232910.043051:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232910.618924:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232910.618932:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232910.618936:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232910.618945:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5783 00000100:00080000:3.0:1761232910.618948:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232910.618954:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232910.618985:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607079936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232910.619006:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232910.619009:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232910.619010:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232910.619012:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5783 00000100:00080000:3.0:1761232910.619014:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232910.619015:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232910.619022:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232910.619028:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607080000/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232910.619034:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232910.619036:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232910.619037:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232910.619039:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5783 00000100:00080000:3.0:1761232910.619040:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232910.619042:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232910.619046:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232910.619053:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607080064/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232910.619058:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232910.619059:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232910.619060:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232910.619062:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5783 00000100:00080000:3.0:1761232910.619063:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232910.619064:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232910.619066:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232910.619071:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607080128/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232910.619131:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607079936:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232910.619241:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232910/real 1761232910] req@0000000090b40c66 x1846780607079936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232935 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232910.619256:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607079936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232935 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232910.619265:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232910.619268:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232910.619271:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607079936:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232910.619276:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080000:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232910.619291:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232910/real 1761232910] req@000000001f8e688e x1846780607080000/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232935 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232910.619298:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607080000/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232935 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232910.619302:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232910.619304:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232910.619306:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080000:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232910.619311:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080064:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232910.619325:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232910/real 1761232910] req@00000000a58ca11e x1846780607080064/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232935 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232910.619332:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607080064/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232935 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232910.619337:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232910.619338:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232910.619339:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080064:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232910.619342:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080128:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232910.619354:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232910/real 1761232910] req@0000000083f16a6b x1846780607080128/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232935 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232910.619359:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607080128/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232935 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232910.619364:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232910.619366:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232910.619367:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080128:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232911.067082:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232912.091059:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232913.115005:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232914.138997:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232915.163036:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232915.739079:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232915.739091:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232915.739096:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232915.739108:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5788 00000100:00080000:0.0:1761232915.739112:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232915.739120:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232915.739165:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607080192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232915.739215:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232915.739219:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232915.739221:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232915.739225:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5788 00000100:00080000:0.0:1761232915.739227:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232915.739230:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232915.739239:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232915.739251:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607080256/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232915.739260:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232915.739264:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232915.739267:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232915.739271:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5788 00000100:00080000:0.0:1761232915.739273:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232915.739277:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232915.739289:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232915.739300:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607080192:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:0.0:1761232915.739304:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607080320/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232915.739319:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232915.739322:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232915.739323:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232915.739326:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5788 00000100:00080000:0.0:1761232915.739329:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232915.739332:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232915.739337:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232915.739350:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607080384/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232915.739411:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232915/real 1761232915] req@00000000b8f2be78 x1846780607080192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232940 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232915.739430:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607080192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232940 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232915.739442:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232915.739445:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232915.739449:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607080192:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232915.739456:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080256:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232915.739480:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232915/real 1761232915] req@00000000fcce404c x1846780607080256/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232940 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232915.739491:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607080256/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232940 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232915.739498:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232915.739500:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232915.739502:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080256:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232915.739557:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080320:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232915.739585:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232915/real 1761232915] req@000000007f60e210 x1846780607080320/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232940 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232915.739597:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607080320/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232940 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232915.739605:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232915.739608:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232915.739611:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080320:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232915.739616:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080384:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232915.739636:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232915/real 1761232915] req@000000000243251f x1846780607080384/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232940 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232915.739646:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607080384/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232940 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232915.739655:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232915.739657:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232915.739659:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080384:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232916.187148:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232917.211083:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232918.235004:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232919.259097:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232920.283079:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232920.859042:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232920.859056:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232920.859062:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232920.859075:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5793 00000100:00080000:1.0:1761232920.859080:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232920.859089:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232920.859138:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607080448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232920.859189:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232920.859198:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232920.859202:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232920.859211:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5793 00000100:00080000:1.0:1761232920.859214:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232920.859221:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232920.859236:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232920.859261:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607080512/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232920.859283:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232920.859288:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232920.859292:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761232920.859295:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607080448:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761232920.859303:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5793 00000100:00080000:1.0:1761232920.859308:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232920.859317:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232920.859324:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232920.859339:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607080576/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232920.859350:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232920.859354:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232920.859356:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232920.859360:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5793 00000100:00080000:1.0:1761232920.859362:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232920.859365:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232920.859370:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232920.859381:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607080640/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761232920.859441:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080512:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761232920.859487:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080576:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761232920.859507:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080640:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232921.307197:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232922.331183:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232923.355154:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761232923.931417:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232920/real 1761232923] req@00000000b72feabb x1846780607080512/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232945 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761232923.931463:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607080512/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232945 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761232923.931482:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232923.931488:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232923.931495:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080512:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761232923.931506:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232920/real 1761232923] req@00000000659d78e3 x1846780607080576/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232945 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761232923.931526:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607080576/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232945 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761232923.931553:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232923.931558:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232923.931563:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080576:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761232923.931573:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232920/real 1761232923] req@00000000db10af3c x1846780607080640/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232945 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761232923.931594:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607080640/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232945 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761232923.931608:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232923.931613:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232923.931618:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080640:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761232923.931682:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232920/real 1761232923] req@00000000d07717c4 x1846780607080448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232945 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761232923.931706:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607080448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232945 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761232923.931719:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232923.931723:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232923.931729:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607080448:192.168.201.104@tcp:250:kworker/u8:1.0 00010000:00010000:2.0:1761232924.379118:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232925.403127:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232925.979015:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232925.979029:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232925.979034:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232925.979049:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5798 00000100:00080000:0.0:1761232925.979055:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232925.979065:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232925.979114:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607080704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232925.979161:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232925.979166:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232925.979168:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232925.979173:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5798 00000100:00080000:0.0:1761232925.979176:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232925.979180:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232925.979191:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232925.979205:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607080768/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232925.979216:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232925.979220:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232925.979221:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232925.979225:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5798 00000100:00080000:0.0:1761232925.979227:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232925.979230:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232925.979234:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232925.979247:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607080832/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232925.979258:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232925.979262:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232925.979263:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232925.979267:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5798 00000100:00080000:0.0:1761232925.979269:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232925.979273:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232925.979277:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232925.979292:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607080704:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:0.0:1761232925.979293:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607080896/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761232925.979439:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232925/real 1761232925] req@00000000271a9d8e x1846780607080704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232950 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761232925.979470:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607080704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232950 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761232925.979487:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232925.979493:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232925.979500:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607080704:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761232925.979511:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080768:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761232925.979574:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232925/real 1761232925] req@0000000001b2f813 x1846780607080768/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232950 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761232925.979644:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607080768/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232950 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761232925.979669:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232925.979674:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232925.979680:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080768:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761232925.979692:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080832:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761232925.979743:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232925/real 1761232925] req@0000000023ab3a3f x1846780607080832/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232950 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761232925.979763:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607080832/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232950 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761232925.979777:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232925.979795:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232925.979801:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080832:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761232925.979896:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080896:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761232925.979951:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232925/real 1761232925] req@0000000099514377 x1846780607080896/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232950 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761232925.979970:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607080896/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232950 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761232925.979984:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232925.979988:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232925.979993:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607080896:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232926.427152:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232927.451095:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232928.475067:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232929.499039:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232930.522969:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232931.098975:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232931.098986:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232931.098990:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232931.099001:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5803 00000100:00080000:3.0:1761232931.099005:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232931.099011:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232931.099064:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607080960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232931.099097:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232931.099102:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232931.099104:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232931.099110:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5803 00000100:00080000:3.0:1761232931.099114:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232931.099119:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232931.099129:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232931.099143:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607081024/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232931.099156:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232931.099162:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232931.099165:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232931.099170:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5803 00000100:00080000:3.0:1761232931.099174:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232931.099178:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232931.099184:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232931.099191:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607080960:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761232931.099201:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607081088/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232931.099218:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232931.099223:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232931.099225:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232931.099232:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5803 00000100:00080000:3.0:1761232931.099236:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232931.099241:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232931.099247:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232931.099265:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607081152/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761232931.099294:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232931/real 1761232931] req@000000007361665d x1846780607080960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232956 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761232931.099314:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607080960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232956 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761232931.099325:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232931.099328:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232931.099332:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607080960:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761232931.099338:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081024:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761232931.099360:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232931/real 1761232931] req@00000000f87d0618 x1846780607081024/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232956 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761232931.099386:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607081024/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232956 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761232931.099397:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232931.099400:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232931.099404:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081024:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761232931.099440:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081088:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761232931.099482:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232931/real 1761232931] req@00000000c7e4e432 x1846780607081088/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232956 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761232931.099500:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607081088/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232956 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761232931.099512:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232931.099516:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232931.099520:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081088:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761232931.099527:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081152:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761232931.099588:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232931/real 1761232931] req@0000000031e043f4 x1846780607081152/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232956 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761232931.099608:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607081152/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232956 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761232931.099622:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232931.099626:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232931.099632:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081152:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232931.547003:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232932.571034:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232933.595101:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232934.619201:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232935.642968:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232936.219030:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232936.219040:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232936.219043:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232936.219052:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5808 00000100:00080000:2.0:1761232936.219056:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232936.219062:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232936.219095:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607081216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232936.219123:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232936.219127:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232936.219129:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232936.219134:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5808 00000100:00080000:2.0:1761232936.219136:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232936.219140:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232936.219167:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232936.219187:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607081280/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232936.219204:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232936.219209:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232936.219212:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232936.219218:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607081216:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761232936.219218:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5808 00000100:00080000:2.0:1761232936.219222:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232936.219228:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232936.219235:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232936.219252:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607081344/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232936.219265:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232936.219271:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232936.219273:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232936.219278:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5808 00000100:00080000:2.0:1761232936.219281:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232936.219286:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232936.219292:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232936.219305:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607081408/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232936.219339:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232936/real 1761232936] req@0000000017386078 x1846780607081216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232961 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232936.219366:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607081216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232961 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232936.219400:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232936.219404:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232936.219410:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607081216:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232936.219420:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081280:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232936.219456:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232936/real 1761232936] req@00000000f38f27df x1846780607081280/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232961 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232936.219520:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607081280/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232961 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232936.219526:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232936.219528:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232936.219530:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081280:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232936.219551:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081344:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232936.219570:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232936/real 1761232936] req@00000000c069b612 x1846780607081344/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232961 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232936.219577:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607081344/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232961 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232936.219583:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232936.219585:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232936.219587:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081344:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232936.219590:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081408:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232936.219602:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232936/real 1761232936] req@0000000064978143 x1846780607081408/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232961 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232936.219609:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607081408/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232961 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232936.219615:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232936.219617:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232936.219618:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081408:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232936.667005:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232937.691071:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232938.715082:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232939.739057:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232940.763073:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232941.338961:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232941.338970:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232941.338973:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232941.338983:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5813 00000100:00080000:3.0:1761232941.338986:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232941.338991:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232941.339022:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607081472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232941.339046:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232941.339049:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232941.339050:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232941.339053:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5813 00000100:00080000:3.0:1761232941.339054:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232941.339057:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232941.339064:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232941.339071:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607081536/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232941.339077:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232941.339079:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232941.339080:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232941.339082:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5813 00000100:00080000:3.0:1761232941.339083:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232941.339084:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232941.339087:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232941.339093:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607081600/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232941.339098:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232941.339100:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232941.339101:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232941.339103:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5813 00000100:00080000:3.0:1761232941.339104:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232941.339105:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232941.339107:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232941.339113:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607081664/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232941.339141:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607081472:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232941.339217:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232941/real 1761232941] req@0000000086ccb527 x1846780607081472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232966 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232941.339228:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607081472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232966 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232941.339235:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232941.339237:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232941.339240:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607081472:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232941.339244:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081536:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232941.339257:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232941/real 1761232941] req@00000000bd684ccc x1846780607081536/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232966 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232941.339263:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607081536/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232966 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232941.339268:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232941.339270:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232941.339271:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081536:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232941.339274:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081600:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232941.339286:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232941/real 1761232941] req@0000000018a8f5da x1846780607081600/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232966 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232941.339292:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607081600/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232966 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232941.339296:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232941.339297:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232941.339298:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081600:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232941.339301:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081664:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232941.339311:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232941/real 1761232941] req@000000005a9bdefd x1846780607081664/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232966 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232941.339318:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607081664/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232966 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232941.339322:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232941.339323:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232941.339325:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081664:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232941.787090:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232942.811043:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232943.835055:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232944.858999:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232945.883055:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761232946.459018:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232946.459029:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761232946.459033:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232946.459044:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5818 00000100:00080000:0.0:1761232946.459047:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232946.459055:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761232946.459090:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607081728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232946.459117:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232946.459121:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761232946.459122:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232946.459126:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5818 00000100:00080000:0.0:1761232946.459128:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232946.459131:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232946.459139:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761232946.459151:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607081792/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232946.459158:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232946.459161:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761232946.459162:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761232946.459165:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5818 00000100:00080000:0.0:1761232946.459167:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232946.459169:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232946.459173:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232946.459182:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607081856/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761232946.459192:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761232946.459195:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761232946.459197:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232946.459201:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607081728:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761232946.459204:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5818 00000100:00080000:0.0:1761232946.459210:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761232946.459213:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761232946.459218:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761232946.459230:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607081920/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232946.459282:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232946/real 1761232946] req@000000002ef1f54f x1846780607081728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232971 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232946.459324:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607081728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232971 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232946.459334:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232946.459338:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232946.459342:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607081728:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232946.459349:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081792:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232946.459378:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232946/real 1761232946] req@00000000275aeee5 x1846780607081792/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232971 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232946.459391:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607081792/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232971 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232946.459398:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232946.459401:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232946.459403:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081792:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232946.459429:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081856:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232946.459456:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232946/real 1761232946] req@000000005a096598 x1846780607081856/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232971 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232946.459468:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607081856/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232971 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232946.459477:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232946.459479:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232946.459482:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081856:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232946.459488:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081920:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232946.459510:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232946/real 1761232946] req@0000000099514377 x1846780607081920/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232971 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232946.459520:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607081920/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232971 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232946.459528:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232946.459530:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232946.459532:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607081920:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232946.907170:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232947.931057:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232948.955044:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232949.978969:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232951.003043:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232951.578983:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232951.578993:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232951.578997:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232951.579008:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5823 00000100:00080000:1.0:1761232951.579011:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232951.579017:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232951.579054:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607081984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232951.579082:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232951.579086:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232951.579087:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232951.579091:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5823 00000100:00080000:1.0:1761232951.579093:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232951.579096:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232951.579104:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232951.579115:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607082048/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232951.579123:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232951.579126:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232951.579127:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232951.579131:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5823 00000100:00080000:1.0:1761232951.579132:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232951.579135:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232951.579139:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232951.579147:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607082112/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232951.579157:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232951.579160:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232951.579162:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761232951.579164:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607081984:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761232951.579168:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5823 00000100:00080000:1.0:1761232951.579172:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232951.579176:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232951.579180:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232951.579190:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607082176/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761232951.579256:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232951/real 1761232951] req@00000000d07717c4 x1846780607081984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232976 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761232951.579269:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607081984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232976 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761232951.579276:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232951.579278:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232951.579281:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607081984:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761232951.579286:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082048:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761232951.579301:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232951/real 1761232951] req@00000000db10af3c x1846780607082048/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232976 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761232951.579308:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607082048/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232976 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761232951.579312:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232951.579314:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232951.579315:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082048:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761232951.579331:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082112:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761232951.579344:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232951/real 1761232951] req@00000000659d78e3 x1846780607082112/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232976 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761232951.579350:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607082112/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232976 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761232951.579355:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232951.579356:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232951.579357:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082112:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761232951.579360:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082176:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761232951.579370:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232951/real 1761232951] req@00000000b72feabb x1846780607082176/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232976 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761232951.579376:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607082176/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232976 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761232951.579381:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232951.579382:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232951.579384:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082176:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232952.027037:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232953.051055:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232954.075037:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232955.099090:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232956.122998:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232956.698920:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232956.698932:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232956.698937:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232956.698951:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5829 00000100:00080000:3.0:1761232956.698956:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232956.698964:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232956.699023:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607082240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232956.699045:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232956.699048:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232956.699050:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232956.699053:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5829 00000100:00080000:3.0:1761232956.699055:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232956.699058:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232956.699069:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232956.699081:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607082304/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761232956.699088:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607082240:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761232956.699094:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232956.699103:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232956.699105:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232956.699108:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5829 00000100:00080000:3.0:1761232956.699110:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232956.699112:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232956.699115:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232956.699127:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607082368/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232956.699132:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232956.699135:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232956.699136:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232956.699152:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5829 00000100:00080000:3.0:1761232956.699154:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232956.699159:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232956.699164:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232956.699179:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607082432/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761232956.699205:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232956/real 1761232956] req@00000000daa39bb1 x1846780607082240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232981 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761232956.699219:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607082240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232981 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761232956.699227:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232956.699229:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232956.699232:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607082240:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761232956.699251:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082304:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761232956.699269:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232956/real 1761232956] req@000000003ad149cc x1846780607082304/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232981 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761232956.699276:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607082304/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232981 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761232956.699280:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232956.699282:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232956.699283:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082304:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761232956.699287:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082368:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761232956.699300:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232956/real 1761232956] req@0000000088473b3e x1846780607082368/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232981 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761232956.699306:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607082368/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232981 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761232956.699311:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232956.699312:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232956.699314:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082368:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761232956.699317:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082432:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761232956.699327:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232956/real 1761232956] req@000000004e3f612a x1846780607082432/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232981 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761232956.699333:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607082432/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232981 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761232956.699338:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232956.699339:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232956.699341:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082432:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232957.147046:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232958.170985:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232959.195069:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232960.219077:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232961.242977:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232961.818985:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232961.818996:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232961.818999:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232961.819010:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5834 00000100:00080000:2.0:1761232961.819012:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232961.819018:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232961.819049:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607082496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232961.819105:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232961.819109:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232961.819110:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232961.819114:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5834 00000100:00080000:2.0:1761232961.819117:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232961.819120:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232961.819129:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232961.819141:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607082560/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232961.819151:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232961.819155:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232961.819157:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232961.819160:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5834 00000100:00080000:2.0:1761232961.819163:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232961.819166:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1761232961.819171:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607082496:192.168.201.104@tcp:250:kworker/u8:2.0 00000008:00100000:2.0:1761232961.819172:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232961.819190:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607082624/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232961.819201:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232961.819204:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232961.819205:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232961.819207:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5834 00000100:00080000:2.0:1761232961.819209:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232961.819212:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232961.819216:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232961.819224:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607082688/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232961.819252:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232961/real 1761232961] req@0000000091106a97 x1846780607082496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232986 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232961.819271:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607082496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232986 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232961.819278:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232961.819280:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232961.819282:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607082496:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232961.819287:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082560:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232961.819302:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232961/real 1761232961] req@0000000094135f44 x1846780607082560/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232986 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232961.819308:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607082560/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232986 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232961.819313:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232961.819314:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232961.819315:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082560:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232961.819332:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082624:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232961.819345:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232961/real 1761232961] req@0000000064978143 x1846780607082624/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232986 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232961.819351:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607082624/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232986 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232961.819357:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232961.819358:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232961.819359:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082624:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232961.819363:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082688:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232961.819373:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232961/real 1761232961] req@00000000c069b612 x1846780607082688/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232986 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232961.819379:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607082688/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232986 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232961.819384:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232961.819385:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232961.819386:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082688:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232962.267002:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232963.290990:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232964.314980:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232965.338983:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232966.363029:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232966.938995:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232966.939004:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232966.939008:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232966.939017:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5839 00000100:00080000:3.0:1761232966.939020:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232966.939025:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232966.939058:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607082752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232966.939082:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232966.939085:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232966.939086:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232966.939089:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5839 00000100:00080000:3.0:1761232966.939090:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232966.939092:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232966.939098:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232966.939104:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607082816/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232966.939109:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232966.939111:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232966.939112:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232966.939115:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5839 00000100:00080000:3.0:1761232966.939116:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232966.939118:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232966.939121:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232966.939127:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607082880/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232966.939132:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232966.939135:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232966.939137:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232966.939139:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5839 00000100:00080000:3.0:1761232966.939140:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232966.939142:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232966.939145:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232966.939146:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607082752:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761232966.939152:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607082944/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232966.939237:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232966/real 1761232966] req@000000004e3f612a x1846780607082752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232991 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232966.939261:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607082752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232991 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232966.939269:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232966.939271:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232966.939274:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607082752:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232966.939279:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082816:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232966.939299:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232966/real 1761232966] req@0000000088473b3e x1846780607082816/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232991 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232966.939306:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607082816/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232991 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232966.939315:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232966.939317:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232966.939318:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082816:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232966.939322:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082880:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232966.939333:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232966/real 1761232966] req@000000003ad149cc x1846780607082880/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232991 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232966.939338:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607082880/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232991 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232966.939344:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232966.939345:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232966.939346:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082880:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232966.939366:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082944:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232966.939378:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232966/real 1761232966] req@00000000daa39bb1 x1846780607082944/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232991 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232966.939384:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607082944/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232991 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232966.939388:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232966.939389:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232966.939391:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607082944:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232967.387027:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232968.411052:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232969.434991:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232970.459024:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232971.482961:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232972.058957:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232972.058966:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232972.058969:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232972.058977:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5844 00000100:00080000:2.0:1761232972.058980:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232972.058985:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232972.059016:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607083008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232972.059038:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232972.059041:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232972.059042:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232972.059045:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5844 00000100:00080000:2.0:1761232972.059046:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232972.059048:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232972.059054:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232972.059061:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607083072/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232972.059067:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232972.059070:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232972.059071:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232972.059073:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5844 00000100:00080000:2.0:1761232972.059075:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232972.059077:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232972.059080:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232972.059088:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607083136/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232972.059094:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232972.059096:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1761232972.059098:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607083008:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:2.0:1761232972.059098:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232972.059104:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5844 00000100:00080000:2.0:1761232972.059107:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232972.059110:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232972.059113:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232972.059119:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607083200/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232972.059190:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232972/real 1761232972] req@00000000f38f27df x1846780607083008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232997 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232972.059204:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607083008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761232997 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232972.059214:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232972.059217:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232972.059221:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607083008:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232972.059228:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083072:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232972.059256:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232972/real 1761232972] req@0000000017386078 x1846780607083072/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232997 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232972.059264:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607083072/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761232997 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232972.059269:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232972.059271:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232972.059273:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083072:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232972.059291:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083136:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232972.059308:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232972/real 1761232972] req@00000000c069b612 x1846780607083136/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232997 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232972.059314:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607083136/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232997 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232972.059319:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232972.059321:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232972.059323:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083136:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232972.059326:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083200:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232972.059337:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232972/real 1761232972] req@0000000064978143 x1846780607083200/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232997 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232972.059342:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607083200/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761232997 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232972.059347:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232972.059348:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232972.059349:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083200:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232972.507008:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232973.531017:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232974.555072:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232975.579028:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232976.603012:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232977.178991:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232977.179000:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232977.179003:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232977.179012:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5849 00000100:00080000:3.0:1761232977.179015:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232977.179020:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232977.179052:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607083264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232977.179081:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232977.179085:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232977.179086:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232977.179090:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5849 00000100:00080000:3.0:1761232977.179092:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232977.179107:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232977.179116:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232977.179129:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607083328/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232977.179139:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232977.179142:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1761232977.179145:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607083264:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:3.0:1761232977.179145:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232977.179152:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5849 00000100:00080000:3.0:1761232977.179155:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232977.179158:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232977.179163:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232977.179177:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607083392/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232977.179197:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232977.179200:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232977.179202:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232977.179205:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5849 00000100:00080000:3.0:1761232977.179207:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232977.179211:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232977.179215:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232977.179226:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607083456/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232977.179236:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232977/real 1761232977] req@000000005a9bdefd x1846780607083264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233002 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232977.179250:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607083264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233002 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232977.179258:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232977.179260:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232977.179263:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607083264:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232977.179281:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083328:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232977.179297:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232977/real 1761232977] req@0000000018a8f5da x1846780607083328/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233002 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232977.179308:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607083328/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233002 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232977.179313:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232977.179314:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232977.179316:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083328:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232977.179319:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083392:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232977.179331:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232977/real 1761232977] req@00000000bd684ccc x1846780607083392/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233002 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232977.179337:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607083392/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233002 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232977.179342:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232977.179343:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232977.179344:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083392:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232977.179347:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083456:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232977.179357:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232977/real 1761232977] req@0000000086ccb527 x1846780607083456/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233002 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232977.179363:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607083456/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233002 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232977.179368:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232977.179369:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232977.179371:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083456:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232977.627088:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232978.651114:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232979.675114:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232980.699017:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232981.723023:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761232982.298910:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232982.298921:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761232982.298925:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232982.298936:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5854 00000100:00080000:3.0:1761232982.298940:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232982.298948:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761232982.298985:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607083520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232982.299009:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232982.299012:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761232982.299014:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232982.299022:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5854 00000100:00080000:3.0:1761232982.299025:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232982.299028:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232982.299037:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761232982.299048:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607083584/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232982.299058:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232982.299060:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761232982.299061:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232982.299064:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607083520:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761232982.299065:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5854 00000100:00080000:3.0:1761232982.299067:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232982.299069:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232982.299073:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232982.299083:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607083648/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761232982.299089:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761232982.299091:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761232982.299093:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761232982.299096:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5854 00000100:00080000:3.0:1761232982.299098:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761232982.299100:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761232982.299103:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761232982.299110:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607083712/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232982.299145:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232982/real 1761232982] req@0000000031e043f4 x1846780607083520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233007 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232982.299162:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607083520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233007 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232982.299169:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232982.299171:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232982.299174:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607083520:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761232982.299192:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083584:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232982.299212:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232982/real 1761232982] req@00000000c7e4e432 x1846780607083584/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233007 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232982.299218:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607083584/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233007 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232982.299223:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232982.299224:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232982.299226:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083584:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761232982.299229:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083648:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232982.299240:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232982/real 1761232982] req@00000000f87d0618 x1846780607083648/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233007 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232982.299253:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607083648/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233007 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232982.299259:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232982.299260:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232982.299262:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083648:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232982.299264:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083712:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761232982.299274:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232982/real 1761232982] req@000000007361665d x1846780607083712/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233007 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761232982.299280:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607083712/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233007 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761232982.299284:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232982.299285:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232982.299287:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083712:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761232982.746976:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232983.771079:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232984.795051:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232985.819004:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232986.843142:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232987.418962:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232987.418975:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232987.418979:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232987.418992:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5859 00000100:00080000:2.0:1761232987.418996:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232987.419005:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232987.419055:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607083776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232987.419084:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232987.419087:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232987.419089:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232987.419092:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5859 00000100:00080000:2.0:1761232987.419094:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232987.419096:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232987.419105:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232987.419113:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607083840/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232987.419119:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232987.419122:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232987.419123:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232987.419125:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5859 00000100:00080000:2.0:1761232987.419126:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232987.419128:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232987.419131:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232987.419137:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607083904/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232987.419141:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232987.419143:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232987.419144:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232987.419148:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5859 00000100:00080000:2.0:1761232987.419150:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232987.419153:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232987.419156:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232987.419162:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607083968/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232987.419183:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607083776:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232987.419286:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083840:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232987.419305:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083904:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232987.419323:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083968:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232987.867139:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232988.891213:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232989.915063:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761232990.491420:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232987/real 1761232990] req@0000000094135f44 x1846780607083776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233012 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232990.491461:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607083776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233012 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232990.491487:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232990.491492:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232990.491498:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607083776:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232990.491508:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232987/real 1761232990] req@0000000091106a97 x1846780607083840/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233012 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232990.491526:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607083840/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233012 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232990.491536:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232990.491539:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232990.491543:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083840:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232990.491551:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232987/real 1761232990] req@0000000064978143 x1846780607083904/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233012 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232990.491566:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607083904/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233012 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232990.491576:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232990.491580:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232990.491584:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083904:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232990.491592:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232987/real 1761232990] req@00000000c069b612 x1846780607083968/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233012 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232990.491654:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607083968/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233012 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232990.491673:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232990.491676:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232990.491680:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607083968:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232990.939171:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232991.963057:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761232992.538973:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232992.538983:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761232992.538987:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232992.538996:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5864 00000100:00080000:1.0:1761232992.538999:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232992.539004:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761232992.539033:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607084032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232992.539055:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232992.539058:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761232992.539059:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232992.539061:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5864 00000100:00080000:1.0:1761232992.539063:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232992.539065:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232992.539071:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761232992.539079:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607084096/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232992.539086:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761232992.539089:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607084032:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761232992.539090:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761232992.539092:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232992.539098:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5864 00000100:00080000:1.0:1761232992.539100:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232992.539101:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232992.539105:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232992.539113:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607084160/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761232992.539118:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761232992.539120:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761232992.539121:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761232992.539123:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5864 00000100:00080000:1.0:1761232992.539124:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761232992.539126:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761232992.539128:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761232992.539134:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607084224/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761232992.539168:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232992/real 1761232992] req@00000000b72feabb x1846780607084032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233017 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761232992.539180:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607084032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233017 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761232992.539188:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232992.539190:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232992.539193:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607084032:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761232992.539211:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084096:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761232992.539226:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232992/real 1761232992] req@00000000659d78e3 x1846780607084096/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233017 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761232992.539233:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607084096/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233017 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761232992.539237:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232992.539239:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232992.539240:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084096:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761232992.539244:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084160:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761232992.539256:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232992/real 1761232992] req@00000000db10af3c x1846780607084160/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233017 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761232992.539262:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607084160/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233017 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761232992.539267:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232992.539268:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232992.539269:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084160:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761232992.539272:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084224:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761232992.539282:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232992/real 1761232992] req@00000000d07717c4 x1846780607084224/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233017 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761232992.539288:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607084224/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233017 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761232992.539292:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761232992.539293:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761232992.539295:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084224:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232992.986993:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232994.011067:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232995.035124:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232996.059140:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761232997.083058:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761232997.658951:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232997.658960:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761232997.658963:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232997.658971:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5870 00000100:00080000:2.0:1761232997.658974:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232997.658979:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761232997.659009:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607084288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232997.659032:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232997.659035:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761232997.659036:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232997.659038:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5870 00000100:00080000:2.0:1761232997.659039:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232997.659041:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232997.659048:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761232997.659058:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607084352/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232997.659064:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232997.659067:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761232997.659068:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761232997.659071:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607084288:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:2.0:1761232997.659075:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5870 00000100:00080000:2.0:1761232997.659080:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232997.659082:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232997.659085:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232997.659094:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607084416/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761232997.659099:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761232997.659101:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761232997.659102:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761232997.659105:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5870 00000100:00080000:2.0:1761232997.659107:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761232997.659109:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761232997.659112:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761232997.659118:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607084480/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761232997.659154:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232997/real 1761232997] req@0000000017386078 x1846780607084288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233022 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232997.659166:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607084288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233022 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232997.659173:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232997.659176:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232997.659178:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607084288:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761232997.659198:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084352:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232997.659215:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232997/real 1761232997] req@00000000f38f27df x1846780607084352/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233022 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232997.659221:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607084352/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233022 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232997.659225:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232997.659227:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232997.659228:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084352:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761232997.659231:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084416:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232997.659243:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232997/real 1761232997] req@00000000c069b612 x1846780607084416/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233022 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232997.659249:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607084416/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233022 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232997.659254:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232997.659255:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232997.659256:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084416:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232997.659259:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084480:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761232997.659269:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761232997/real 1761232997] req@0000000064978143 x1846780607084480/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233022 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761232997.659274:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607084480/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233022 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761232997.659280:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761232997.659281:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761232997.659283:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084480:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761232998.106983:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761232999.130986:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233000.155077:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233001.179023:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233002.203020:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233002.778950:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233002.778960:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233002.778963:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233002.778973:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5875 00000100:00080000:2.0:1761233002.778976:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233002.778982:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233002.779014:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607084544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233002.779039:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233002.779044:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233002.779045:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233002.779050:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5875 00000100:00080000:2.0:1761233002.779053:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233002.779057:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233002.779066:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233002.779084:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607084608/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233002.779094:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607084544:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761233002.779098:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233002.779104:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233002.779106:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233002.779111:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5875 00000100:00080000:2.0:1761233002.779114:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233002.779118:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233002.779123:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233002.779135:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607084672/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233002.779141:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233002.779144:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233002.779145:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233002.779147:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5875 00000100:00080000:2.0:1761233002.779148:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233002.779150:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233002.779152:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233002.779160:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607084736/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233002.779198:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233002/real 1761233002] req@0000000091106a97 x1846780607084544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233027 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233002.779219:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607084544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233027 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233002.779230:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233002.779233:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233002.779237:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607084544:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233002.779263:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084608:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233002.779294:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233002/real 1761233002] req@0000000094135f44 x1846780607084608/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233027 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233002.779303:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607084608/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233027 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233002.779308:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233002.779310:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233002.779312:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084608:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233002.779316:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084672:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233002.779331:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233002/real 1761233002] req@0000000064978143 x1846780607084672/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233027 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233002.779338:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607084672/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233027 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233002.779343:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233002.779344:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233002.779346:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084672:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233002.779349:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084736:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233002.779361:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233002/real 1761233002] req@00000000c069b612 x1846780607084736/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233027 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233002.779368:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607084736/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233027 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233002.779373:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233002.779374:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233002.779376:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084736:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233003.227020:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233004.250997:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233005.275029:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233006.299050:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233007.323066:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233007.899073:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233007.899087:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233007.899093:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233007.899106:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5880 00000100:00080000:3.0:1761233007.899110:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233007.899119:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233007.899170:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607084800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233007.899201:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233007.899204:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233007.899206:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233007.899210:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5880 00000100:00080000:3.0:1761233007.899212:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233007.899215:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233007.899225:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233007.899235:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607084864/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233007.899242:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233007.899246:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233007.899247:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233007.899251:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5880 00000100:00080000:3.0:1761233007.899253:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233007.899255:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233007.899258:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233007.899268:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607084928/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233007.899275:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233007.899278:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233007.899279:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233007.899282:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5880 00000100:00080000:3.0:1761233007.899284:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233007.899287:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233007.899290:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233007.899301:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607084800:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761233007.899301:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607084992/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233007.899462:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233007/real 1761233007] req@0000000083f16a6b x1846780607084800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233032 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233007.899480:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607084800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233032 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233007.899496:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233007.899499:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233007.899502:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607084800:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233007.899508:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084864:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233007.899528:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233007/real 1761233007] req@00000000a58ca11e x1846780607084864/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233032 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233007.899537:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607084864/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233032 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233007.899542:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233007.899544:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233007.899546:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084864:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233007.899550:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084928:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233007.899565:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233007/real 1761233007] req@000000001f8e688e x1846780607084928/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233032 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233007.899573:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607084928/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233032 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233007.899580:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233007.899581:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233007.899583:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084928:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233007.899611:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084992:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233007.899627:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233007/real 1761233007] req@0000000090b40c66 x1846780607084992/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233032 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233007.899635:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607084992/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233032 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233007.899642:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233007.899644:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233007.899645:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607084992:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233008.347173:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233009.371014:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233010.395070:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233011.419022:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233012.443090:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233013.018972:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233013.018985:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233013.018990:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233013.019003:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5885 00000100:00080000:2.0:1761233013.019007:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233013.019016:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233013.019075:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607085056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233013.019107:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233013.019110:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233013.019112:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233013.019116:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5885 00000100:00080000:2.0:1761233013.019118:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233013.019120:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233013.019129:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233013.019141:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607085120/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233013.019148:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233013.019150:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233013.019152:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233013.019155:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5885 00000100:00080000:2.0:1761233013.019157:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233013.019161:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233013.019167:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233013.019180:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607085184/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233013.019192:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233013.019197:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233013.019199:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761233013.019204:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607085056:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761233013.019206:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5885 00000100:00080000:2.0:1761233013.019212:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233013.019219:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233013.019225:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233013.019240:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607085248/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233013.019355:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233013/real 1761233013] req@00000000f38f27df x1846780607085056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233038 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233013.019372:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607085056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233038 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233013.019382:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233013.019385:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233013.019388:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607085056:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233013.019393:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085120:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233013.019412:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233013/real 1761233013] req@0000000017386078 x1846780607085120/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233038 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233013.019420:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607085120/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233038 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233013.019430:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233013.019432:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233013.019433:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085120:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233013.019458:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085184:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233013.019473:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233013/real 1761233013] req@00000000c069b612 x1846780607085184/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233038 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233013.019480:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607085184/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233038 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233013.019486:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233013.019487:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233013.019488:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085184:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233013.019493:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085248:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233013.019505:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233013/real 1761233013] req@0000000064978143 x1846780607085248/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233038 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233013.019513:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607085248/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233038 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233013.019519:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233013.019520:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233013.019521:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085248:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233013.467109:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233014.490997:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233015.515181:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233016.539052:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233017.563018:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233018.138927:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233018.138936:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233018.138939:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233018.138948:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5890 00000100:00080000:2.0:1761233018.138951:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233018.138957:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233018.138991:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607085312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233018.139013:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233018.139016:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233018.139017:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233018.139020:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5890 00000100:00080000:2.0:1761233018.139021:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233018.139023:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233018.139029:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233018.139036:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607085376/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233018.139042:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233018.139046:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233018.139047:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233018.139049:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5890 00000100:00080000:2.0:1761233018.139050:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233018.139052:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233018.139054:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233018.139060:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607085440/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233018.139064:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233018.139066:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233018.139066:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233018.139068:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5890 00000100:00080000:2.0:1761233018.139069:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233018.139071:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233018.139074:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233018.139079:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607085504/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233018.139134:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607085312:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233018.139298:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233018/real 1761233018] req@0000000094135f44 x1846780607085312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233043 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233018.139319:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607085312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233043 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233018.139331:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233018.139334:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233018.139339:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607085312:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233018.139347:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085376:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233018.139376:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233018/real 1761233018] req@0000000091106a97 x1846780607085376/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233043 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233018.139387:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607085376/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233043 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233018.139395:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233018.139398:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233018.139401:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085376:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233018.139408:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085440:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233018.139432:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233018/real 1761233018] req@0000000064978143 x1846780607085440/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233043 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233018.139444:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607085440/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233043 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233018.139459:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233018.139461:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233018.139465:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085440:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233018.139471:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085504:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233018.139493:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233018/real 1761233018] req@00000000c069b612 x1846780607085504/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233043 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233018.139504:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607085504/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233043 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233018.139513:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233018.139515:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233018.139518:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085504:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233018.586999:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233019.610982:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233020.635049:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233021.659053:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233022.682986:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233023.258915:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233023.258924:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233023.258927:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233023.258936:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5895 00000100:00080000:3.0:1761233023.258938:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233023.258943:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233023.258971:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607085568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233023.258990:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233023.258992:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233023.258993:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233023.258996:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5895 00000100:00080000:3.0:1761233023.258998:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233023.259000:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233023.259005:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233023.259013:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607085632/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233023.259019:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233023.259021:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233023.259022:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233023.259025:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5895 00000100:00080000:3.0:1761233023.259026:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233023.259028:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233023.259030:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233023.259036:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607085568:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761233023.259040:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607085696/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233023.259049:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233023.259051:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233023.259051:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233023.259054:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5895 00000100:00080000:3.0:1761233023.259055:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233023.259057:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233023.259060:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233023.259068:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607085760/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233023.259141:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233023/real 1761233023] req@000000006572b591 x1846780607085568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233048 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233023.259159:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607085568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233048 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233023.259169:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233023.259172:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233023.259176:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607085568:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233023.259182:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085632:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233023.259212:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233023/real 1761233023] req@00000000fc01c41c x1846780607085632/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233048 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233023.259224:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607085632/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233048 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233023.259232:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233023.259234:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233023.259237:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085632:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233023.259261:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085696:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233023.259289:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233023/real 1761233023] req@00000000408f088c x1846780607085696/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233048 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233023.259300:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607085696/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233048 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233023.259308:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233023.259311:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233023.259313:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085696:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233023.259319:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085760:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233023.259341:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233023/real 1761233023] req@000000002ce934d3 x1846780607085760/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233048 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233023.259352:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607085760/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233048 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233023.259360:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233023.259363:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233023.259366:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085760:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233023.706999:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233024.730944:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233025.755079:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233026.778959:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233027.803114:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233028.378979:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233028.378988:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233028.378993:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233028.379002:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5900 00000100:00080000:0.0:1761233028.379005:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233028.379030:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233028.379068:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607085824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233028.379096:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233028.379099:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233028.379101:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233028.379104:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5900 00000100:00080000:0.0:1761233028.379106:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233028.379108:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233028.379115:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233028.379126:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607085888/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233028.379132:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233028.379135:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233028.379137:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233028.379140:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5900 00000100:00080000:0.0:1761233028.379142:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233028.379144:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233028.379147:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233028.379154:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607085952/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233028.379161:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607085824:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761233028.379162:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233028.379167:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233028.379169:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233028.379173:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5900 00000100:00080000:0.0:1761233028.379174:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233028.379175:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233028.379178:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233028.379185:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607086016/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233028.379256:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233028/real 1761233028] req@0000000023ab3a3f x1846780607085824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233053 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233028.379275:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607085824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233053 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233028.379286:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233028.379290:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233028.379294:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607085824:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233028.379302:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085888:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233028.379334:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233028/real 1761233028] req@0000000001b2f813 x1846780607085888/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233053 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233028.379347:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607085888/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233053 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233028.379355:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233028.379358:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233028.379361:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085888:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233028.379388:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085952:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233028.379418:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233028/real 1761233028] req@00000000271a9d8e x1846780607085952/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233053 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233028.379432:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607085952/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233053 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233028.379441:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233028.379443:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233028.379447:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607085952:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233028.379453:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086016:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233028.379479:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233028/real 1761233028] req@000000000243251f x1846780607086016/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233053 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233028.379490:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607086016/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233053 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233028.379498:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233028.379501:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233028.379504:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086016:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233028.827026:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233029.850964:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233030.874987:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233031.899051:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233032.923044:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233033.498928:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233033.498949:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233033.498952:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233033.498960:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5905 00000100:00080000:1.0:1761233033.498963:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233033.498969:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233033.498999:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607086080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233033.499023:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233033.499026:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233033.499027:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233033.499030:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5905 00000100:00080000:1.0:1761233033.499032:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233033.499034:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233033.499040:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233033.499048:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607086144/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233033.499053:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607086080:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761233033.499057:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233033.499060:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233033.499061:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233033.499064:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5905 00000100:00080000:1.0:1761233033.499065:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233033.499067:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233033.499069:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233033.499079:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607086208/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233033.499084:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233033.499086:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233033.499087:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233033.499089:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5905 00000100:00080000:1.0:1761233033.499090:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233033.499092:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233033.499095:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233033.499101:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607086272/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233033.499129:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233033/real 1761233033] req@00000000d07717c4 x1846780607086080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233058 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233033.499142:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607086080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233058 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233033.499150:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233033.499152:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233033.499155:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607086080:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233033.499171:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086144:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233033.499187:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233033/real 1761233033] req@00000000db10af3c x1846780607086144/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233058 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233033.499193:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607086144/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233058 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233033.499197:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233033.499199:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233033.499201:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086144:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233033.499204:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086208:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233033.499215:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233033/real 1761233033] req@00000000659d78e3 x1846780607086208/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233058 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233033.499221:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607086208/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233058 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233033.499225:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233033.499226:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233033.499228:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086208:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233033.499230:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086272:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233033.499241:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233033/real 1761233033] req@00000000b72feabb x1846780607086272/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233058 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233033.499246:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607086272/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233058 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233033.499251:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233033.499252:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233033.499254:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086272:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233033.947045:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233034.970967:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233035.995069:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233037.019027:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233038.043114:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233038.618960:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233038.618969:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233038.618972:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233038.618981:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5911 00000100:00080000:3.0:1761233038.618984:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233038.618990:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233038.619022:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607086336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233038.619043:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233038.619046:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233038.619047:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233038.619050:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5911 00000100:00080000:3.0:1761233038.619052:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233038.619054:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233038.619060:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233038.619067:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607086400/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233038.619074:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233038.619076:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233038.619077:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233038.619081:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5911 00000100:00080000:3.0:1761233038.619082:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233038.619084:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233038.619087:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233038.619097:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607086336:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761233038.619097:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607086464/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233038.619110:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233038.619112:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233038.619112:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233038.619115:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5911 00000100:00080000:3.0:1761233038.619116:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233038.619117:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233038.619121:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233038.619128:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607086528/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233038.619179:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233038/real 1761233038] req@000000002ce934d3 x1846780607086336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233063 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233038.619192:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607086336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233063 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233038.619199:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233038.619201:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233038.619203:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607086336:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233038.619207:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086400:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233038.619221:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233038/real 1761233038] req@00000000408f088c x1846780607086400/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233063 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233038.619227:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607086400/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233063 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233038.619235:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233038.619236:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233038.619238:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086400:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233038.619255:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086464:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233038.619267:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233038/real 1761233038] req@00000000fc01c41c x1846780607086464/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233063 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233038.619273:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607086464/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233063 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233038.619278:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233038.619279:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233038.619280:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086464:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233038.619283:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086528:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233038.619293:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233038/real 1761233038] req@000000006572b591 x1846780607086528/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233063 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233038.619299:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607086528/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233063 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233038.619304:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233038.619305:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233038.619306:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086528:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233039.067056:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233040.091050:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233041.115066:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233042.139024:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233043.163001:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233043.738932:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233043.738941:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233043.738944:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233043.738952:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5916 00000100:00080000:3.0:1761233043.738955:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233043.738961:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233043.738993:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607086592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233043.739015:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233043.739018:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233043.739019:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233043.739021:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5916 00000100:00080000:3.0:1761233043.739023:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233043.739025:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233043.739031:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233043.739038:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607086656/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233043.739043:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233043.739045:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233043.739046:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233043.739048:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5916 00000100:00080000:3.0:1761233043.739050:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233043.739053:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233043.739055:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233043.739062:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607086720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233043.739067:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233043.739069:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233043.739070:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233043.739072:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5916 00000100:00080000:3.0:1761233043.739073:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233043.739079:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1761233043.739080:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607086592:192.168.201.104@tcp:250:kworker/u8:1.0 00000008:00100000:3.0:1761233043.739082:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233043.739091:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607086784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233043.739170:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233043/real 1761233043] req@0000000090b40c66 x1846780607086592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233068 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233043.739187:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607086592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233068 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233043.739197:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233043.739200:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233043.739204:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607086592:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233043.739213:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086656:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233043.739241:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233043/real 1761233043] req@000000001f8e688e x1846780607086656/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233068 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233043.739251:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607086656/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233068 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233043.739256:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233043.739258:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233043.739260:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086656:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233043.739263:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086720:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233043.739279:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233043/real 1761233043] req@00000000a58ca11e x1846780607086720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233068 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233043.739285:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607086720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233068 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233043.739294:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233043.739296:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233043.739297:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086720:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233043.739317:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086784:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233043.739329:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233043/real 1761233043] req@0000000083f16a6b x1846780607086784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233068 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233043.739334:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607086784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233068 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233043.739339:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233043.739340:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233043.739341:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086784:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233044.186971:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233045.210981:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233046.235012:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233047.259149:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233048.283037:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233048.858995:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233048.859012:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233048.859018:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233048.859033:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5921 00000100:00080000:3.0:1761233048.859039:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233048.859049:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233048.859102:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607086848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233048.859147:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233048.859155:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233048.859159:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233048.859166:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5921 00000100:00080000:3.0:1761233048.859170:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233048.859177:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233048.859191:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233048.859215:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607086912/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233048.859223:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607086848:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:3.0:1761233048.859248:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233048.859256:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233048.859259:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233048.859268:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5921 00000100:00080000:3.0:1761233048.859272:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233048.859280:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233048.859291:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233048.859315:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607086976/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233048.859333:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761233048.859339:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233048/real 1761233048] req@000000007361665d x1846780607086848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233073 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761233048.859345:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233048.859349:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233048.859363:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5921 00000100:00080000:3.0:1761233048.859369:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1761233048.859374:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607086848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233073 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:3.0:1761233048.859379:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233048.859394:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00080000:2.0:1761233048.859398:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233048.859404:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233048.859412:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607086848:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761233048.859420:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607087040/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233048.859446:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086912:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233048.859486:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233048/real 1761233048] req@00000000f87d0618 x1846780607086912/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233073 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233048.859516:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607086912/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233073 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233048.859525:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233048.859528:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233048.859531:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086912:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233048.859537:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086976:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233048.859569:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233048/real 1761233048] req@00000000c7e4e432 x1846780607086976/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233073 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233048.859581:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607086976/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233073 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233048.859590:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233048.859592:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233048.859595:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607086976:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233048.859611:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087040:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233048.859632:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233048/real 1761233048] req@0000000031e043f4 x1846780607087040/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233073 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233048.859643:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607087040/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233073 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233048.859651:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233048.859654:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233048.859657:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087040:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233049.307047:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233050.330929:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233051.355014:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233052.379151:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233053.403124:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233053.978914:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233053.978924:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233053.978928:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233053.978938:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5926 00000100:00080000:3.0:1761233053.978941:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233053.978947:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233053.978980:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607087104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233053.979005:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233053.979008:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233053.979009:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233053.979013:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5926 00000100:00080000:3.0:1761233053.979015:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233053.979018:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233053.979025:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233053.979033:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607087168/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233053.979046:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233053.979048:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233053.979049:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233053.979052:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5926 00000100:00080000:3.0:1761233053.979053:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233053.979056:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233053.979059:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233053.979084:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607087232/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233053.979092:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233053.979095:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233053.979097:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761233053.979112:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607087104:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:3.0:1761233053.979112:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5926 00000100:00080000:3.0:1761233053.979116:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233053.979123:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233053.979130:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233053.979140:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607087296/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233053.979237:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087168:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233053.979262:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087232:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233053.979295:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087296:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233054.427024:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233055.451043:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233056.475077:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761233057.051374:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233053/real 1761233057] req@000000005a9bdefd x1846780607087296/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233078 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233057.051411:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607087296/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233078 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233057.051425:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233057.051430:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233057.051435:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087296:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233057.051442:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233053/real 1761233057] req@0000000086ccb527 x1846780607087104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233078 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233057.051454:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607087104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233078 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233057.051462:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233057.051464:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233057.051466:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607087104:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233057.051473:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233053/real 1761233057] req@00000000bd684ccc x1846780607087168/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233078 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233057.051483:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607087168/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233078 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233057.051489:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233057.051492:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233057.051495:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087168:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233057.051502:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233053/real 1761233057] req@0000000018a8f5da x1846780607087232/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233078 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233057.051513:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607087232/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233078 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233057.051522:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233057.051524:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233057.051527:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087232:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233057.499027:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233058.523045:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233059.099001:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233059.099012:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233059.099016:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233059.099026:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5931 00000100:00080000:1.0:1761233059.099030:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233059.099036:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233059.099070:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607087360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233059.099096:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233059.099099:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233059.099101:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233059.099104:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5931 00000100:00080000:1.0:1761233059.099106:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233059.099109:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233059.099117:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233059.099125:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607087424/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233059.099137:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233059.099140:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233059.099142:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233059.099145:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5931 00000100:00080000:1.0:1761233059.099147:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233059.099149:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233059.099169:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233059.099180:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607087488/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233059.099188:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233059.099191:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233059.099193:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233059.099196:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5931 00000100:00080000:1.0:1761233059.099197:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233059.099200:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233059.099203:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233059.099213:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607087552/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233059.099232:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607087360:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233059.099381:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233059/real 1761233059] req@00000000b72feabb x1846780607087360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233084 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233059.099405:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607087360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233084 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233059.099418:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233059.099423:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233059.099429:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607087360:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233059.099437:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087424:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233059.099470:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233059/real 1761233059] req@00000000659d78e3 x1846780607087424/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233084 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233059.099482:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607087424/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233084 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233059.099489:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233059.099492:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233059.099495:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087424:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233059.099502:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087488:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233059.099547:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233059/real 1761233059] req@00000000db10af3c x1846780607087488/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233084 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233059.099560:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607087488/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233084 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233059.099569:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233059.099572:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233059.099576:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087488:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233059.099615:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087552:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233059.099643:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233059/real 1761233059] req@00000000d07717c4 x1846780607087552/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233084 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233059.099656:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607087552/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233084 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233059.099666:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233059.099669:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233059.099672:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087552:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233059.547082:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233060.571068:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233061.595000:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233062.619148:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233063.643004:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233064.219173:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233064.219191:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233064.219198:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233064.219217:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5936 00000100:00080000:2.0:1761233064.219223:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233064.219234:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233064.219297:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607087616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233064.219345:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233064.219351:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233064.219353:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233064.219358:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5936 00000100:00080000:2.0:1761233064.219361:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233064.219366:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233064.219379:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233064.219393:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607087680/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233064.219404:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233064.219408:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233064.219410:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233064.219417:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5936 00000100:00080000:2.0:1761233064.219420:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233064.219426:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233064.219432:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233064.219476:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607087744/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233064.219506:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233064.219510:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233064.219512:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761233064.219516:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607087616:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761233064.219521:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5936 00000100:00080000:2.0:1761233064.219529:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233064.219532:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233064.219538:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233064.219553:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607087808/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233064.219663:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233064/real 1761233064] req@0000000017386078 x1846780607087616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233089 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233064.219685:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607087616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233089 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233064.219699:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233064.219702:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233064.219707:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607087616:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233064.219717:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087680:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233064.219748:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233064/real 1761233064] req@00000000f38f27df x1846780607087680/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233089 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233064.219759:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607087680/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233089 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233064.219766:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233064.219769:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233064.219771:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087680:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233064.219803:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087744:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233064.219825:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233064/real 1761233064] req@00000000c069b612 x1846780607087744/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233089 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233064.219839:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607087744/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233089 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233064.219915:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233064.219919:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233064.219923:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087744:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233064.219930:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087808:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233064.219966:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233064/real 1761233064] req@0000000064978143 x1846780607087808/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233089 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233064.219978:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607087808/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233089 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233064.219989:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233064.219993:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233064.219997:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087808:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233064.667186:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233065.690976:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233066.715154:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233067.738978:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233068.763007:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233069.339028:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233069.339044:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233069.339051:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233069.339067:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5941 00000100:00080000:1.0:1761233069.339072:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233069.339094:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233069.339153:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607087872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233069.339191:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233069.339195:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233069.339197:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233069.339202:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5941 00000100:00080000:1.0:1761233069.339205:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233069.339208:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233069.339221:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233069.339248:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607087936/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233069.339261:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761233069.339267:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607087872:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761233069.339269:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233069.339273:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233069.339282:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5941 00000100:00080000:1.0:1761233069.339285:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233069.339288:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233069.339293:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233069.339306:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607088000/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233069.339314:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233069.339318:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233069.339320:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233069.339324:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5941 00000100:00080000:1.0:1761233069.339327:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233069.339330:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233069.339334:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233069.339345:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607088064/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233069.339436:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233069/real 1761233069] req@00000000d07717c4 x1846780607087872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233094 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233069.339460:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607087872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233094 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233069.339473:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233069.339476:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233069.339481:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607087872:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233069.339518:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087936:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233069.339549:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233069/real 1761233069] req@00000000db10af3c x1846780607087936/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233094 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233069.339560:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607087936/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233094 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233069.339568:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233069.339570:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233069.339572:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607087936:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233069.339578:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088000:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233069.339600:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233069/real 1761233069] req@00000000659d78e3 x1846780607088000/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233094 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233069.339610:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607088000/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233094 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233069.339618:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233069.339620:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233069.339623:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088000:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233069.339629:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088064:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233069.339647:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233069/real 1761233069] req@00000000b72feabb x1846780607088064/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233094 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233069.339657:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607088064/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233094 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233069.339666:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233069.339668:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233069.339671:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088064:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233069.787151:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233070.811056:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233071.835117:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233072.859239:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233073.883110:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233074.459055:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233074.459068:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233074.459072:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233074.459083:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5946 00000100:00080000:3.0:1761233074.459087:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233074.459094:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233074.459136:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607088128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233074.459173:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233074.459178:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233074.459181:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233074.459187:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5946 00000100:00080000:3.0:1761233074.459191:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233074.459195:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233074.459208:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233074.459226:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607088192/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233074.459240:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233074.459245:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233074.459247:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233074.459253:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5946 00000100:00080000:3.0:1761233074.459256:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233074.459261:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233074.459270:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233074.459324:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607088256/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233074.459335:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233074.459339:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233074.459341:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233074.459344:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5946 00000100:00080000:3.0:1761233074.459346:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233074.459350:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233074.459357:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233074.459369:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607088320/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233074.459385:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607088128:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233074.459573:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233074/real 1761233074] req@00000000daa39bb1 x1846780607088128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233099 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233074.459599:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607088128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233099 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233074.459614:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233074.459618:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233074.459624:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607088128:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233074.459634:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088192:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233074.459667:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233074/real 1761233074] req@000000003ad149cc x1846780607088192/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233099 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233074.459680:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607088192/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233099 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233074.459688:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233074.459691:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233074.459694:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088192:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233074.459701:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088256:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233074.459727:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233074/real 1761233074] req@0000000088473b3e x1846780607088256/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233099 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233074.459739:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607088256/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233099 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233074.459749:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233074.459751:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233074.459754:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088256:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233074.459796:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088320:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233074.459820:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233074/real 1761233074] req@000000004e3f612a x1846780607088320/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233099 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233074.459906:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607088320/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233099 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233074.459917:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233074.459920:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233074.459923:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088320:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233074.907037:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233075.930980:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233076.955131:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233077.979080:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233079.003026:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233079.578917:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233079.578929:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233079.578933:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233079.578943:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5951 00000100:00080000:3.0:1761233079.578946:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233079.578953:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233079.578993:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607088384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233079.579024:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233079.579027:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233079.579028:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233079.579031:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5951 00000100:00080000:3.0:1761233079.579033:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233079.579036:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233079.579043:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233079.579053:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607088448/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233079.579058:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233079.579061:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233079.579062:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233079.579064:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5951 00000100:00080000:3.0:1761233079.579066:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233079.579068:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233079.579071:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233079.579078:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607088512/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233079.579084:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233079.579086:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233079.579087:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233079.579089:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5951 00000100:00080000:3.0:1761233079.579090:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233079.579092:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233079.579094:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233079.579097:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607088384:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:3.0:1761233079.579103:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607088576/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233079.579197:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233079/real 1761233079] req@000000004e3f612a x1846780607088384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233104 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233079.579251:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607088384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233104 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233079.579261:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233079.579266:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233079.579270:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607088384:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233079.579278:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088448:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233079.579308:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233079/real 1761233079] req@0000000088473b3e x1846780607088448/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233104 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233079.579320:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607088448/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233104 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233079.579327:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233079.579330:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233079.579333:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088448:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233079.579339:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088512:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233079.579363:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233079/real 1761233079] req@000000003ad149cc x1846780607088512/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233104 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233079.579374:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607088512/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233104 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233079.579383:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233079.579385:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233079.579388:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088512:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233079.579415:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088576:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233079.579439:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233079/real 1761233079] req@00000000daa39bb1 x1846780607088576/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233104 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233079.579457:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607088576/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233104 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233079.579465:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233079.579468:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233079.579471:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088576:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233080.027008:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233081.050949:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233082.075153:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233083.099011:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233084.123087:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233084.698966:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233084.698979:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233084.698985:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233084.698999:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5957 00000100:00080000:1.0:1761233084.699003:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233084.699022:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233084.699065:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607088640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233084.699098:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233084.699102:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233084.699104:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233084.699108:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5957 00000100:00080000:1.0:1761233084.699110:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233084.699113:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233084.699121:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233084.699153:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607088704/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233084.699163:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233084.699194:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233084.699196:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233084.699199:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5957 00000100:00080000:1.0:1761233084.699200:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233084.699208:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1761233084.699209:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607088640:192.168.201.104@tcp:250:kworker/u8:2.0 00000008:00100000:1.0:1761233084.699213:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233084.699224:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607088768/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233084.699233:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233084.699236:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233084.699237:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233084.699240:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5957 00000100:00080000:1.0:1761233084.699242:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233084.699245:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233084.699248:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233084.699256:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607088832/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233084.699300:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233084/real 1761233084] req@00000000b72feabb x1846780607088640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233109 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233084.699317:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607088640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233109 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233084.699325:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233084.699328:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233084.699331:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607088640:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233084.699337:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088704:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233084.699356:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233084/real 1761233084] req@00000000659d78e3 x1846780607088704/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233109 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233084.699364:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607088704/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233109 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233084.699370:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233084.699372:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233084.699374:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088704:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233084.699406:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088768:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233084.699423:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233084/real 1761233084] req@00000000db10af3c x1846780607088768/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233109 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233084.699430:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607088768/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233109 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233084.699436:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233084.699438:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233084.699440:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088768:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233084.699443:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088832:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233084.699456:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233084/real 1761233084] req@00000000d07717c4 x1846780607088832/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233109 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233084.699463:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607088832/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233109 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233084.699469:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233084.699471:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233084.699473:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088832:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233085.147006:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233086.171176:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233087.195022:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233088.219014:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233089.243139:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233089.818974:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233089.818998:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233089.819002:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233089.819011:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5962 00000100:00080000:2.0:1761233089.819014:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233089.819019:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233089.819064:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607088896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233089.819113:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233089.819115:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233089.819116:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233089.819119:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5962 00000100:00080000:2.0:1761233089.819121:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233089.819123:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233089.819130:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233089.819140:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607088960/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233089.819147:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607088896:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761233089.819148:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233089.819155:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233089.819157:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233089.819160:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5962 00000100:00080000:2.0:1761233089.819161:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233089.819163:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233089.819166:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233089.819176:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607089024/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233089.819181:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233089.819183:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233089.819184:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233089.819187:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5962 00000100:00080000:2.0:1761233089.819188:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233089.819191:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233089.819195:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233089.819213:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607089088/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233089.819228:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233089/real 1761233089] req@0000000091106a97 x1846780607088896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233114 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233089.819242:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607088896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233114 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233089.819250:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233089.819252:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233089.819254:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607088896:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233089.819272:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088960:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233089.819288:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233089/real 1761233089] req@0000000094135f44 x1846780607088960/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233114 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233089.819295:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607088960/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233114 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233089.819299:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233089.819301:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233089.819303:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607088960:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233089.819306:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089024:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233089.819318:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233089/real 1761233089] req@0000000064978143 x1846780607089024/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233114 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233089.819324:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607089024/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233114 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233089.819329:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233089.819330:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233089.819331:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089024:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233089.819334:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089088:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233089.819344:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233089/real 1761233089] req@00000000c069b612 x1846780607089088/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233114 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233089.819350:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607089088/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233114 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233089.819354:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233089.819355:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233089.819357:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089088:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233090.267122:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233091.290991:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233092.314998:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233093.339010:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233094.363049:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233094.938920:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233094.938929:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233094.938932:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233094.938940:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5967 00000100:00080000:2.0:1761233094.938943:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233094.938948:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233094.939015:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607089152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233094.939036:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233094.939038:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233094.939039:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233094.939042:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5967 00000100:00080000:2.0:1761233094.939043:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233094.939045:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233094.939051:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233094.939059:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607089216/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233094.939065:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233094.939067:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233094.939067:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233094.939070:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5967 00000100:00080000:2.0:1761233094.939071:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233094.939072:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233094.939074:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233094.939082:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607089280/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233094.939086:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233094.939087:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233094.939088:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233094.939090:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5967 00000100:00080000:2.0:1761233094.939092:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233094.939094:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233094.939097:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233094.939102:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607089344/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233094.939117:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607089152:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233094.939192:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233094/real 1761233094] req@00000000f38f27df x1846780607089152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233119 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233094.939205:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607089152/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233119 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233094.939212:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233094.939225:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233094.939228:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607089152:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233094.939232:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089216:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233094.939246:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233094/real 1761233094] req@0000000017386078 x1846780607089216/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233119 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233094.939252:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607089216/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233119 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233094.939257:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233094.939258:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233094.939259:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089216:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233094.939263:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089280:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233094.939274:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233094/real 1761233094] req@00000000c069b612 x1846780607089280/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233119 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233094.939281:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607089280/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233119 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233094.939286:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233094.939287:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233094.939288:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089280:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233094.939307:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089344:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233094.939319:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233094/real 1761233094] req@0000000064978143 x1846780607089344/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233119 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233094.939325:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607089344/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233119 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233094.939330:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233094.939331:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233094.939332:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089344:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233095.386993:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233096.411008:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233097.435029:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233098.459005:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233099.483047:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233100.059039:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233100.059052:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233100.059058:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233100.059071:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5972 00000100:00080000:2.0:1761233100.059076:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233100.059084:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233100.059132:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607089408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233100.059176:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233100.059183:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233100.059187:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233100.059195:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5972 00000100:00080000:2.0:1761233100.059200:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233100.059207:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233100.059221:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233100.059247:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607089472/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233100.059252:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607089408:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761233100.059277:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233100.059285:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233100.059288:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233100.059296:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5972 00000100:00080000:2.0:1761233100.059301:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233100.059309:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233100.059317:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233100.059336:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607089536/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233100.059347:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233100.059353:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233100.059355:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233100.059360:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5972 00000100:00080000:2.0:1761233100.059363:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233100.059368:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1761233100.059373:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233100/real 1761233100] req@0000000094135f44 x1846780607089408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233125 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000008:00100000:2.0:1761233100.059374:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233100.059396:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607089600/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233100.059404:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607089408/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233125 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233100.059424:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233100.059427:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233100.059433:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607089408:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233100.059461:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089472:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233100.059490:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233100/real 1761233100] req@0000000091106a97 x1846780607089472/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233125 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233100.059501:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607089472/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233125 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233100.059510:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233100.059512:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233100.059515:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089472:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233100.059521:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089536:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233100.059541:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233100/real 1761233100] req@0000000064978143 x1846780607089536/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233125 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233100.059552:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607089536/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233125 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233100.059560:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233100.059562:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233100.059564:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089536:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233100.059569:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089600:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233100.059587:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233100/real 1761233100] req@00000000c069b612 x1846780607089600/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233125 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233100.059598:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607089600/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233125 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233100.059606:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233100.059608:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233100.059610:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089600:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233100.507096:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233101.531017:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233102.554996:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233103.579109:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233104.603024:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233105.178980:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233105.178989:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233105.178992:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233105.179001:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5977 00000100:00080000:2.0:1761233105.179005:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233105.179011:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233105.179047:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607089664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233105.179067:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233105.179070:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233105.179071:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233105.179074:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5977 00000100:00080000:2.0:1761233105.179075:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233105.179077:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233105.179083:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233105.179091:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607089728/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233105.179097:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233105.179099:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233105.179100:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233105.179102:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5977 00000100:00080000:2.0:1761233105.179103:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233105.179105:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233105.179108:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233105.179114:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607089792/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233105.179118:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233105.179120:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233105.179120:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233105.179122:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5977 00000100:00080000:2.0:1761233105.179123:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233105.179125:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233105.179127:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233105.179132:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607089856/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233105.179164:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607089664:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233105.179254:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233105/real 1761233105] req@0000000017386078 x1846780607089664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233130 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233105.179270:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607089664/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233130 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233105.179280:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233105.179283:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233105.179287:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607089664:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233105.179293:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089728:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233105.179314:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233105/real 1761233105] req@00000000f38f27df x1846780607089728/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233130 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233105.179321:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607089728/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233130 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233105.179326:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233105.179327:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233105.179329:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089728:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233105.179332:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089792:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233105.179344:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233105/real 1761233105] req@00000000c069b612 x1846780607089792/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233130 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233105.179350:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607089792/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233130 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233105.179358:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233105.179360:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233105.179361:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089792:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233105.179364:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089856:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233105.179374:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233105/real 1761233105] req@0000000064978143 x1846780607089856/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233130 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233105.179380:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607089856/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233130 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233105.179384:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233105.179386:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233105.179387:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089856:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233105.627074:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233106.650926:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233107.675169:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233108.699045:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233109.723077:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233110.298990:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233110.299001:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233110.299005:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233110.299016:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5982 00000100:00080000:0.0:1761233110.299020:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233110.299026:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233110.299077:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607089920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233110.299116:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233110.299120:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233110.299121:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233110.299125:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5982 00000100:00080000:0.0:1761233110.299127:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233110.299130:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233110.299140:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233110.299157:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607089984/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233110.299169:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761233110.299172:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607089920:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761233110.299174:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233110.299177:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233110.299185:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5982 00000100:00080000:0.0:1761233110.299187:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233110.299191:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233110.299196:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233110.299210:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607090048/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233110.299218:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233110.299222:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233110.299224:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233110.299227:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5982 00000100:00080000:0.0:1761233110.299229:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233110.299232:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233110.299237:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233110.299249:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607090112/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233110.299273:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233110/real 1761233110] req@000000007f60e210 x1846780607089920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233135 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233110.299293:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607089920/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233135 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233110.299319:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233110.299321:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233110.299326:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607089920:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233110.299349:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089984:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233110.299375:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233110/real 1761233110] req@00000000fcce404c x1846780607089984/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233135 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233110.299386:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607089984/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233135 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233110.299394:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233110.299396:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233110.299399:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607089984:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233110.299405:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090048:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233110.299430:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233110/real 1761233110] req@00000000b8f2be78 x1846780607090048/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233135 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233110.299442:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607090048/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233135 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233110.299449:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233110.299452:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233110.299455:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090048:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233110.299460:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090112:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233110.299485:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233110/real 1761233110] req@000000000243251f x1846780607090112/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233135 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233110.299496:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607090112/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233135 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233110.299504:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233110.299507:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233110.299509:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090112:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233110.747069:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233111.771042:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233112.795137:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233113.819134:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233114.842957:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233115.418963:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233115.418974:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233115.418977:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233115.418986:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5987 00000100:00080000:0.0:1761233115.418989:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233115.418996:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233115.419031:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607090176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233115.419056:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233115.419059:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233115.419061:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233115.419065:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5987 00000100:00080000:0.0:1761233115.419067:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233115.419070:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233115.419079:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233115.419096:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607090240/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233115.419104:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233115.419107:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233115.419108:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233115.419112:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5987 00000100:00080000:0.0:1761233115.419114:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233115.419118:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1761233115.419121:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607090176:192.168.201.104@tcp:250:kworker/u8:1.0 00000008:00100000:0.0:1761233115.419125:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233115.419137:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607090304/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233115.419146:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233115.419149:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233115.419150:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233115.419153:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5987 00000100:00080000:0.0:1761233115.419155:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233115.419158:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233115.419163:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233115.419172:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607090368/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233115.419215:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233115/real 1761233115] req@00000000271a9d8e x1846780607090176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233140 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233115.419234:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607090176/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233140 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233115.419244:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233115.419248:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233115.419252:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607090176:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233115.419260:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090240:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233115.419289:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233115/real 1761233115] req@0000000001b2f813 x1846780607090240/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233140 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233115.419300:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607090240/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233140 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233115.419308:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233115.419310:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233115.419313:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090240:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233115.419339:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090304:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233115.419363:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233115/real 1761233115] req@0000000023ab3a3f x1846780607090304/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233140 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233115.419374:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607090304/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233140 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233115.419382:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233115.419384:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233115.419386:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090304:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233115.419392:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090368:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233115.419415:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233115/real 1761233115] req@0000000099514377 x1846780607090368/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233140 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233115.419426:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607090368/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233140 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233115.419434:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233115.419436:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233115.419439:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090368:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233115.867049:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233116.890995:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233117.915000:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233118.939074:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233119.963091:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233120.539025:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233120.539049:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233120.539055:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233120.539068:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5992 00000100:00080000:0.0:1761233120.539073:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233120.539081:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233120.539131:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607090432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233120.539165:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233120.539170:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233120.539173:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233120.539178:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5992 00000100:00080000:0.0:1761233120.539181:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233120.539185:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233120.539195:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233120.539209:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607090496/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233120.539221:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233120.539225:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233120.539228:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233120.539234:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5992 00000100:00080000:0.0:1761233120.539237:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233120.539241:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233120.539245:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233120.539259:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607090560/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233120.539270:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607090432:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761233120.539276:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233120.539283:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233120.539285:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233120.539290:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5992 00000100:00080000:0.0:1761233120.539293:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233120.539296:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233120.539301:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233120.539316:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607090624/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233120.539392:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090496:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233120.539437:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090560:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233120.539476:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090624:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233120.987037:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233122.011077:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233123.035107:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761233123.611404:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233120/real 1761233123] req@000000002ef1f54f x1846780607090560/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233145 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233123.611464:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607090560/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233145 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233123.611484:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233123.611490:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233123.611499:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090560:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233123.611510:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233120/real 1761233123] req@0000000099514377 x1846780607090624/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233145 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233123.611588:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607090624/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233145 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233123.611602:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233123.611606:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233123.611611:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090624:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233123.611621:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233120/real 1761233123] req@000000005a096598 x1846780607090432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233145 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233123.611640:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607090432/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233145 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233123.611654:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233123.611658:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233123.611663:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607090432:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233123.611673:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233120/real 1761233123] req@00000000275aeee5 x1846780607090496/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233145 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233123.611716:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607090496/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233145 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233123.611729:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233123.611734:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233123.611739:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090496:192.168.201.104@tcp:38:kworker/u8:1.0 00010000:00010000:0.0:1761233124.058947:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233125.082984:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233125.658955:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233125.658964:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233125.658967:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233125.658976:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 5998 00000100:00080000:2.0:1761233125.658979:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233125.658984:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233125.659021:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607090688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233125.659042:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233125.659044:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233125.659045:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233125.659048:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5998 00000100:00080000:2.0:1761233125.659049:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233125.659052:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233125.659057:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233125.659064:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607090752/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233125.659071:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233125.659073:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233125.659074:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233125.659077:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5998 00000100:00080000:2.0:1761233125.659078:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233125.659080:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233125.659084:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233125.659091:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607090816/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233125.659097:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607090688:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761233125.659100:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233125.659106:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233125.659108:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233125.659111:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 5998 00000100:00080000:2.0:1761233125.659112:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233125.659113:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233125.659116:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233125.659124:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607090880/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233125.659202:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233125/real 1761233125] req@0000000091106a97 x1846780607090688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233150 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233125.659220:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607090688/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233150 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233125.659232:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233125.659235:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233125.659239:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607090688:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233125.659246:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090752:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233125.659287:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233125/real 1761233125] req@0000000094135f44 x1846780607090752/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233150 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233125.659300:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607090752/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233150 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233125.659307:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233125.659309:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233125.659312:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090752:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233125.659337:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090816:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233125.659366:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233125/real 1761233125] req@0000000064978143 x1846780607090816/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233150 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233125.659379:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607090816/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233150 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233125.659388:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233125.659390:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233125.659393:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090816:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233125.659399:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090880:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233125.659421:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233125/real 1761233125] req@00000000c069b612 x1846780607090880/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233150 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233125.659433:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607090880/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233150 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233125.659442:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233125.659444:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233125.659446:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607090880:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233126.107170:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233127.130997:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233128.155066:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233129.179031:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233130.202978:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233130.778986:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233130.778999:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233130.779003:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233130.779016:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6003 00000100:00080000:2.0:1761233130.779021:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233130.779029:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233130.779078:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607090944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233130.779113:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233130.779118:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233130.779120:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233130.779125:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6003 00000100:00080000:2.0:1761233130.779129:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233130.779132:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233130.779142:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233130.779155:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607091008/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233130.779166:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233130.779170:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233130.779172:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233130.779176:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6003 00000100:00080000:2.0:1761233130.779178:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233130.779181:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233130.779185:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233130.779195:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607091072/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233130.779205:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233130.779208:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233130.779211:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233130.779216:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6003 00000100:00080000:2.0:1761233130.779219:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233130.779223:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233130.779228:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233130.779238:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607091136/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233130.779255:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607090944:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233130.779397:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233130/real 1761233130] req@00000000f38f27df x1846780607090944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233155 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233130.779449:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607090944/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233155 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233130.779464:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233130.779468:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233130.779472:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607090944:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233130.779481:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091008:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233130.779519:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233130/real 1761233130] req@0000000017386078 x1846780607091008/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233155 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233130.779532:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607091008/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233155 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233130.779540:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233130.779542:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233130.779545:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091008:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233130.779552:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091072:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233130.779575:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233130/real 1761233130] req@00000000c069b612 x1846780607091072/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233155 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233130.779586:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607091072/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233155 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233130.779596:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233130.779599:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233130.779601:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091072:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233130.779633:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091136:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233130.779657:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233130/real 1761233130] req@0000000064978143 x1846780607091136/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233155 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233130.779668:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607091136/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233155 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233130.779677:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233130.779680:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233130.779682:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091136:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233131.227033:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233132.251043:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233133.275149:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233134.299049:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233135.323013:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233135.898970:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233135.898979:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233135.898982:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233135.898991:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6008 00000100:00080000:0.0:1761233135.898994:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233135.899000:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233135.899031:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607091200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233135.899054:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233135.899057:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233135.899058:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233135.899061:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6008 00000100:00080000:0.0:1761233135.899064:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233135.899066:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233135.899072:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233135.899081:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607091264/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233135.899086:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233135.899088:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233135.899089:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233135.899091:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6008 00000100:00080000:0.0:1761233135.899092:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233135.899094:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233135.899096:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233135.899101:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607091328/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233135.899105:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233135.899107:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233135.899109:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233135.899111:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6008 00000100:00080000:0.0:1761233135.899113:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233135.899116:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233135.899118:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233135.899123:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607091392/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233135.899138:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607091200:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233135.899211:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233135/real 1761233135] req@0000000023ab3a3f x1846780607091200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233160 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233135.899223:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607091200/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233160 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233135.899230:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233135.899232:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233135.899235:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607091200:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233135.899239:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091264:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233135.899253:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233135/real 1761233135] req@0000000001b2f813 x1846780607091264/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233160 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233135.899259:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607091264/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233160 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233135.899263:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233135.899266:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233135.899267:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091264:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233135.899271:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091328:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233135.899282:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233135/real 1761233135] req@00000000271a9d8e x1846780607091328/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233160 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233135.899287:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607091328/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233160 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233135.899292:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233135.899294:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233135.899295:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091328:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233135.899313:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091392:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233135.899343:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233135/real 1761233135] req@000000000243251f x1846780607091392/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233160 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233135.899351:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607091392/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233160 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233135.899356:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233135.899358:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233135.899359:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091392:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233136.347116:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233137.370951:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233138.395050:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233139.419002:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233140.443073:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233141.019023:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233141.019037:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233141.019042:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233141.019055:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6013 00000100:00080000:3.0:1761233141.019059:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233141.019067:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233141.019112:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607091456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233141.019164:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233141.019169:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233141.019171:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233141.019177:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6013 00000100:00080000:3.0:1761233141.019180:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233141.019185:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233141.019197:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233141.019214:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607091520/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233141.019228:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761233141.019234:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607091456:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:3.0:1761233141.019235:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233141.019264:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233141.019276:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6013 00000100:00080000:3.0:1761233141.019279:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233141.019284:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233141.019303:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233141.019324:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607091584/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233141.019336:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233141.019340:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233141.019342:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233141.019348:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6013 00000100:00080000:3.0:1761233141.019351:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233141.019355:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233141.019361:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233141.019378:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607091648/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233141.019385:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233141/real 1761233141] req@0000000018a8f5da x1846780607091456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233166 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233141.019425:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607091456/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233166 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233141.019437:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233141.019440:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233141.019444:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607091456:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233141.019468:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091520:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233141.019492:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233141/real 1761233141] req@00000000bd684ccc x1846780607091520/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233166 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233141.019513:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607091520/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233166 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233141.019520:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233141.019522:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233141.019525:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091520:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233141.019529:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091584:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233141.019546:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233141/real 1761233141] req@0000000086ccb527 x1846780607091584/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233166 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233141.019555:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607091584/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233166 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233141.019562:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233141.019564:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233141.019567:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091584:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233141.019571:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091648:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233141.019587:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233141/real 1761233141] req@000000005a9bdefd x1846780607091648/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233166 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233141.019596:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607091648/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233166 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233141.019603:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233141.019605:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233141.019607:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091648:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233141.467053:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233142.491022:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233143.515002:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233144.539053:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233145.563106:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233146.139056:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233146.139069:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233146.139073:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233146.139085:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6018 00000100:00080000:0.0:1761233146.139090:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233146.139098:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233146.139153:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607091712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233146.139245:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233146.139249:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233146.139251:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233146.139256:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6018 00000100:00080000:0.0:1761233146.139259:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233146.139263:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233146.139273:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233146.139289:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607091776/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233146.139301:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233146.139306:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233146.139308:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233146.139312:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6018 00000100:00080000:0.0:1761233146.139314:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233146.139319:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233146.139324:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233146.139337:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607091712:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:0.0:1761233146.139341:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607091840/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233146.139359:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233146.139363:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233146.139364:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233146.139368:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6018 00000100:00080000:0.0:1761233146.139370:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233146.139374:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233146.139379:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233146.139394:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607091904/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233146.139474:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233146/real 1761233146] req@00000000b8f2be78 x1846780607091712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233171 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233146.139495:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607091712/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233171 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233146.139507:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233146.139511:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233146.139526:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607091712:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233146.139535:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091776:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233146.139561:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233146/real 1761233146] req@00000000fcce404c x1846780607091776/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233171 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233146.139574:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607091776/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233171 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233146.139618:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233146.139621:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233146.139625:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091776:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233146.139656:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091840:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233146.139693:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233146/real 1761233146] req@000000007f60e210 x1846780607091840/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233171 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233146.139705:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607091840/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233171 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233146.139714:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233146.139717:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233146.139719:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091840:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233146.139725:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091904:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233146.139746:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233146/real 1761233146] req@000000000243251f x1846780607091904/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233171 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233146.139757:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607091904/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233171 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233146.139766:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233146.139768:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233146.139771:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607091904:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233146.587136:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233147.611045:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233148.635182:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233149.659031:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233150.683138:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233151.258987:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233151.258997:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233151.259000:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233151.259009:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6023 00000100:00080000:2.0:1761233151.259012:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233151.259018:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233151.259050:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607091968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233151.259071:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233151.259074:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233151.259075:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233151.259088:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6023 00000100:00080000:2.0:1761233151.259089:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233151.259092:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233151.259099:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233151.259107:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607092032/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233151.259113:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233151.259115:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233151.259117:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233151.259119:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6023 00000100:00080000:2.0:1761233151.259121:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233151.259123:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233151.259126:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233151.259132:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607092096/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233151.259137:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233151.259139:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233151.259141:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233151.259143:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6023 00000100:00080000:2.0:1761233151.259145:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233151.259147:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233151.259149:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233151.259155:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607092160/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233151.259164:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607091968:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233151.259241:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233151/real 1761233151] req@0000000094135f44 x1846780607091968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233176 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233151.259254:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607091968/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233176 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233151.259261:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233151.259263:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233151.259266:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607091968:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233151.259271:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092032:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233151.259286:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233151/real 1761233151] req@0000000091106a97 x1846780607092032/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233176 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233151.259293:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607092032/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233176 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233151.259298:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233151.259299:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233151.259301:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092032:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233151.259304:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092096:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233151.259316:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233151/real 1761233151] req@0000000064978143 x1846780607092096/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233176 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233151.259323:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607092096/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233176 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233151.259328:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233151.259330:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233151.259331:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092096:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233151.259351:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092160:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233151.259363:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233151/real 1761233151] req@00000000c069b612 x1846780607092160/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233176 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233151.259370:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607092160/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233176 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233151.259374:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233151.259375:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233151.259377:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092160:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233151.706992:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233152.731007:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233153.755009:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233154.779068:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233155.803189:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233156.378916:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233156.378925:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233156.378929:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233156.378938:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6028 00000100:00080000:2.0:1761233156.378941:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233156.378963:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233156.378994:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607092224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233156.379052:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233156.379055:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233156.379056:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233156.379059:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6028 00000100:00080000:2.0:1761233156.379060:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233156.379062:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233156.379069:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233156.379079:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607092288/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233156.379085:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233156.379088:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233156.379089:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233156.379091:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6028 00000100:00080000:2.0:1761233156.379092:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233156.379095:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233156.379097:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233156.379104:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607092224:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233156.379106:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607092352/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233156.379115:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233156.379116:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233156.379117:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233156.379119:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6028 00000100:00080000:2.0:1761233156.379120:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233156.379123:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233156.379125:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233156.379132:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607092416/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233156.379191:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233156/real 1761233156] req@0000000017386078 x1846780607092224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233181 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233156.379205:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607092224/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233181 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233156.379212:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233156.379214:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233156.379217:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607092224:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233156.379221:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092288:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233156.379238:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233156/real 1761233156] req@00000000f38f27df x1846780607092288/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233181 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233156.379250:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607092288/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233181 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233156.379254:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233156.379255:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233156.379256:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092288:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233156.379273:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092352:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233156.379286:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233156/real 1761233156] req@00000000c069b612 x1846780607092352/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233181 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233156.379293:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607092352/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233181 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233156.379298:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233156.379299:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233156.379300:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092352:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233156.379303:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092416:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233156.379314:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233156/real 1761233156] req@0000000064978143 x1846780607092416/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233181 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233156.379320:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607092416/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233181 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233156.379324:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233156.379325:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233156.379328:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092416:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233156.827061:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233157.851019:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233158.875000:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233159.899104:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233160.923037:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233161.499046:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233161.499056:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233161.499059:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233161.499069:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6033 00000100:00080000:3.0:1761233161.499072:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233161.499078:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233161.499109:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607092480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233161.499138:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233161.499146:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233161.499148:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233161.499156:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6033 00000100:00080000:3.0:1761233161.499160:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233161.499166:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233161.499174:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233161.499186:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607092544/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233161.499187:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607092480:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761233161.499221:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233161.499226:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233161.499229:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233161.499235:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6033 00000100:00080000:3.0:1761233161.499238:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233161.499244:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233161.499251:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233161.499262:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607092608/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233161.499267:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233161.499269:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233161.499269:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233161.499272:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6033 00000100:00080000:3.0:1761233161.499274:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233161.499279:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233161.499285:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233161.499298:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607092672/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233161.499311:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233161/real 1761233161] req@0000000031e043f4 x1846780607092480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233186 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233161.499366:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607092480/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233186 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233161.499377:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233161.499381:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233161.499388:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607092480:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233161.499419:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092544:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233161.499439:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233161/real 1761233161] req@00000000c7e4e432 x1846780607092544/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233186 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233161.499454:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607092544/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233186 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233161.499458:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233161.499462:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233161.499466:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092544:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233161.499472:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092608:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233161.499498:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233161/real 1761233161] req@00000000f87d0618 x1846780607092608/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233186 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233161.499511:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607092608/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233186 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233161.499524:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233161.499529:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233161.499533:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092608:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233161.499541:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092672:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233161.499565:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233161/real 1761233161] req@000000007361665d x1846780607092672/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233186 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233161.499573:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607092672/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233186 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233161.499577:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233161.499581:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233161.499583:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092672:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233161.947029:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233162.971069:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233163.995030:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233165.019028:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233166.043064:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233166.619082:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233166.619096:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233166.619101:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233166.619114:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6039 00000100:00080000:0.0:1761233166.619118:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233166.619126:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233166.619170:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607092736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233166.619200:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233166.619204:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233166.619205:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233166.619209:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6039 00000100:00080000:0.0:1761233166.619211:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233166.619213:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233166.619223:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233166.619232:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607092800/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233166.619238:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233166.619240:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233166.619241:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233166.619244:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6039 00000100:00080000:0.0:1761233166.619246:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233166.619249:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233166.619252:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233166.619260:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607092864/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233166.619267:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233166.619271:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233166.619272:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233166.619275:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6039 00000100:00080000:0.0:1761233166.619277:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233166.619280:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233166.619282:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233166.619290:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607092928/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233166.619301:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607092736:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233166.619420:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233166/real 1761233166] req@00000000271a9d8e x1846780607092736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233191 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233166.619442:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607092736/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233191 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233166.619455:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233166.619458:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233166.619462:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607092736:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233166.619468:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092800:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233166.619506:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233166/real 1761233166] req@0000000001b2f813 x1846780607092800/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233191 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233166.619519:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607092800/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233191 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233166.619528:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233166.619530:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233166.619534:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092800:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233166.619540:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092864:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233166.619567:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233166/real 1761233166] req@0000000023ab3a3f x1846780607092864/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233191 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233166.619579:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607092864/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233191 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233166.619588:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233166.619590:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233166.619593:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092864:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233166.619626:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092928:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233166.619655:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233166/real 1761233166] req@00000000275aeee5 x1846780607092928/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233191 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233166.619667:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607092928/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233191 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233166.619677:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233166.619680:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233166.619683:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607092928:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233167.067086:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233168.091106:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233169.115048:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233170.139113:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233171.163023:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233171.739074:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233171.739088:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233171.739092:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233171.739107:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6044 00000100:00080000:1.0:1761233171.739112:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233171.739120:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233171.739170:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607092992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233171.739210:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233171.739215:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233171.739216:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233171.739221:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6044 00000100:00080000:1.0:1761233171.739224:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233171.739227:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233171.739237:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233171.739250:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607093056/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233171.739260:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233171.739264:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233171.739265:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233171.739269:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6044 00000100:00080000:1.0:1761233171.739271:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233171.739276:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233171.739281:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233171.739292:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607093120/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233171.739302:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233171.739307:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233171.739309:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233171.739312:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6044 00000100:00080000:1.0:1761233171.739314:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233171.739318:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233171.739323:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233171.739334:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607092992:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233171.739337:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607093184/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233171.739453:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233171/real 1761233171] req@00000000d07717c4 x1846780607092992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233196 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233171.739475:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607092992/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233196 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233171.739488:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233171.739491:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233171.739496:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607092992:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233171.739504:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093056:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233171.739531:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233171/real 1761233171] req@00000000db10af3c x1846780607093056/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233196 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233171.739543:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607093056/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233196 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233171.739552:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233171.739555:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233171.739558:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093056:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233171.739564:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093120:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233171.739585:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233171/real 1761233171] req@00000000659d78e3 x1846780607093120/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233196 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233171.739596:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607093120/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233196 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233171.739604:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233171.739607:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233171.739609:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093120:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233171.739639:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093184:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233171.739661:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233171/real 1761233171] req@00000000b72feabb x1846780607093184/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233196 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233171.739672:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607093184/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233196 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233171.739681:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233171.739683:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233171.739686:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093184:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233172.187097:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233173.211079:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233174.235104:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233175.259123:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233176.282977:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233176.858984:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233176.858998:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233176.859003:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233176.859016:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6049 00000100:00080000:1.0:1761233176.859020:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233176.859027:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233176.859075:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607093248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233176.859102:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233176.859105:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233176.859106:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233176.859111:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6049 00000100:00080000:1.0:1761233176.859112:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233176.859114:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233176.859123:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233176.859131:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607093312/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233176.859136:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233176.859139:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233176.859140:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233176.859143:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6049 00000100:00080000:1.0:1761233176.859144:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233176.859146:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233176.859149:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233176.859156:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607093376/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233176.859162:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233176.859165:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233176.859166:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233176.859168:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6049 00000100:00080000:1.0:1761233176.859169:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233176.859171:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233176.859173:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233176.859179:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607093440/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233176.859226:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607093248:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233176.859367:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233176/real 1761233176] req@00000000b72feabb x1846780607093248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233201 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233176.859390:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607093248/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233201 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233176.859402:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233176.859406:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233176.859411:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607093248:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233176.859420:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093312:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233176.859450:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233176/real 1761233176] req@00000000659d78e3 x1846780607093312/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233201 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233176.859463:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607093312/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233201 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233176.859471:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233176.859474:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233176.859477:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093312:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233176.859484:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093376:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233176.859507:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233176/real 1761233176] req@00000000db10af3c x1846780607093376/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233201 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233176.859519:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607093376/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233201 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233176.859529:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233176.859532:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233176.859535:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093376:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233176.859568:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093440:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233176.859592:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233176/real 1761233176] req@00000000d07717c4 x1846780607093440/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233201 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233176.859603:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607093440/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233201 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233176.859612:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233176.859614:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233176.859617:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093440:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233177.307064:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233178.331055:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233179.355071:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233180.379083:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233181.403005:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233181.978906:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233181.978916:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233181.978920:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233181.978929:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6054 00000100:00080000:1.0:1761233181.978932:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233181.978938:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233181.978974:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607093504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233181.978995:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233181.978997:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233181.978998:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233181.979001:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6054 00000100:00080000:1.0:1761233181.979002:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233181.979005:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233181.979012:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233181.979019:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607093568/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233181.979025:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233181.979027:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233181.979028:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233181.979031:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6054 00000100:00080000:1.0:1761233181.979032:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233181.979034:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233181.979043:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233181.979050:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607093632/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233181.979055:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233181.979057:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233181.979059:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233181.979061:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6054 00000100:00080000:1.0:1761233181.979062:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233181.979064:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233181.979067:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233181.979074:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607093696/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233181.979092:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607093504:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233181.979170:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233181/real 1761233181] req@00000000d07717c4 x1846780607093504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233206 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233181.979184:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607093504/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233206 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233181.979192:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233181.979195:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233181.979198:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607093504:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233181.979203:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093568:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233181.979218:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233181/real 1761233181] req@00000000db10af3c x1846780607093568/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233206 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233181.979226:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607093568/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233206 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233181.979230:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233181.979232:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233181.979233:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093568:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233181.979237:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093632:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233181.979250:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233181/real 1761233181] req@00000000659d78e3 x1846780607093632/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233206 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233181.979257:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607093632/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233206 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233181.979262:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233181.979263:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233181.979265:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093632:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233181.979268:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093696:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233181.979280:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233181/real 1761233181] req@00000000b72feabb x1846780607093696/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233206 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233181.979286:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607093696/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233206 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233181.979291:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233181.979293:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233181.979295:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093696:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233182.426945:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233183.451136:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233184.475080:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233185.499195:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233186.523053:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233187.098978:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233187.098988:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233187.098991:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233187.099000:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6059 00000100:00080000:3.0:1761233187.099002:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233187.099009:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233187.099040:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607093760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233187.099060:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233187.099063:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233187.099064:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233187.099067:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6059 00000100:00080000:3.0:1761233187.099068:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233187.099070:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233187.099076:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233187.099083:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607093824/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233187.099088:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233187.099090:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233187.099091:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233187.099093:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6059 00000100:00080000:3.0:1761233187.099095:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233187.099096:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233187.099100:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233187.099107:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607093888/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233187.099112:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233187.099115:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233187.099116:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233187.099118:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6059 00000100:00080000:3.0:1761233187.099120:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233187.099121:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233187.099123:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233187.099129:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607093952/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233187.099137:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607093760:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233187.099207:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093824:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233187.099221:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093888:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233187.099239:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093952:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233187.547074:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233188.571038:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233189.595053:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761233190.171422:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233187/real 1761233190] req@0000000090b40c66 x1846780607093952/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233212 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233190.171501:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607093952/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233212 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233190.171522:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233190.171529:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233190.171536:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093952:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233190.171548:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233187/real 1761233190] req@0000000083f16a6b x1846780607093760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233212 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233190.171572:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607093760/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233212 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233190.171585:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233190.171590:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233190.171595:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607093760:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233190.171607:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233187/real 1761233190] req@00000000a58ca11e x1846780607093824/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233212 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233190.171626:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607093824/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233212 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233190.171640:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233190.171645:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233190.171650:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093824:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233190.171659:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233187/real 1761233190] req@000000001f8e688e x1846780607093888/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233212 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233190.171680:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607093888/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233212 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233190.171693:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233190.171697:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233190.171701:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607093888:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761233190.619037:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233191.643100:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233192.219031:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233192.219045:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233192.219050:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233192.219064:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6064 00000100:00080000:0.0:1761233192.219068:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233192.219077:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233192.219120:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607094016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233192.219148:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233192.219151:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233192.219153:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233192.219156:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6064 00000100:00080000:0.0:1761233192.219157:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233192.219160:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233192.219169:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233192.219179:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607094080/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233192.219185:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233192.219187:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233192.219188:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233192.219191:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6064 00000100:00080000:0.0:1761233192.219193:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233192.219197:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233192.219200:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233192.219207:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607094144/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233192.219213:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233192.219216:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233192.219218:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233192.219222:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6064 00000100:00080000:0.0:1761233192.219224:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233192.219226:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233192.219229:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233192.219236:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607094208/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233192.219275:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607094016:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233192.219457:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233192/real 1761233192] req@000000005a096598 x1846780607094016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233217 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233192.219485:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607094016/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233217 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233192.219499:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233192.219504:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233192.219509:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607094016:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233192.219522:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094080:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233192.219579:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233192/real 1761233192] req@0000000099514377 x1846780607094080/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233217 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233192.219602:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607094080/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233217 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233192.219616:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233192.219621:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233192.219626:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094080:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233192.219638:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094144:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233192.219706:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233192/real 1761233192] req@000000002ef1f54f x1846780607094144/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233217 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233192.219728:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607094144/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233217 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233192.219743:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233192.219749:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233192.219754:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094144:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233192.219767:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094208:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233192.219833:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233192/real 1761233192] req@00000000275aeee5 x1846780607094208/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233217 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233192.219946:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607094208/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233217 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233192.219959:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233192.219963:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233192.219969:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094208:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233192.667175:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233193.691020:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233194.715097:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233195.739109:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233196.763024:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233197.338928:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233197.338939:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233197.338942:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233197.338952:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6069 00000100:00080000:0.0:1761233197.338955:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233197.338962:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233197.338998:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607094272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233197.339020:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233197.339023:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233197.339025:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233197.339028:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6069 00000100:00080000:0.0:1761233197.339030:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233197.339032:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233197.339038:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233197.339048:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607094336/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233197.339054:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233197.339058:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233197.339059:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233197.339061:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6069 00000100:00080000:0.0:1761233197.339063:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233197.339065:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233197.339068:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233197.339073:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607094400/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233197.339079:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233197.339081:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233197.339082:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233197.339084:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6069 00000100:00080000:0.0:1761233197.339086:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233197.339089:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233197.339092:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233197.339098:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607094464/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233197.339120:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607094272:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233197.339206:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233197/real 1761233197] req@0000000023ab3a3f x1846780607094272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233222 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233197.339220:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607094272/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233222 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233197.339229:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233197.339231:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233197.339234:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607094272:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233197.339240:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094336:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233197.339257:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233197/real 1761233197] req@0000000001b2f813 x1846780607094336/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233222 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233197.339265:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607094336/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233222 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233197.339275:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233197.339277:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233197.339279:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094336:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233197.339283:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094400:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233197.339296:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233197/real 1761233197] req@00000000271a9d8e x1846780607094400/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233222 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233197.339303:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607094400/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233222 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233197.339309:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233197.339310:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233197.339312:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094400:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233197.339315:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094464:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233197.339328:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233197/real 1761233197] req@000000000243251f x1846780607094464/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233222 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233197.339335:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607094464/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233222 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233197.339341:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233197.339342:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233197.339344:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094464:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233197.786990:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233198.811087:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233199.835156:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233200.859122:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233201.883037:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233202.459063:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233202.459077:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233202.459081:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233202.459094:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6074 00000100:00080000:1.0:1761233202.459099:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233202.459108:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233202.459153:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607094528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233202.459182:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233202.459186:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233202.459187:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233202.459191:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6074 00000100:00080000:1.0:1761233202.459194:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233202.459197:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233202.459205:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233202.459215:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607094592/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233202.459222:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233202.459225:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233202.459226:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233202.459229:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6074 00000100:00080000:1.0:1761233202.459231:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233202.459233:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233202.459236:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233202.459242:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607094656/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233202.459248:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233202.459250:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233202.459251:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233202.459253:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6074 00000100:00080000:1.0:1761233202.459255:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233202.459258:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233202.459262:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233202.459269:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607094720/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233202.459330:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607094528:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233202.459434:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233202/real 1761233202] req@00000000b72feabb x1846780607094528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233227 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233202.459451:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607094528/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233227 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233202.459461:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233202.459463:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233202.459467:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607094528:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233202.459472:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094592:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233202.459491:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233202/real 1761233202] req@00000000659d78e3 x1846780607094592/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233227 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233202.459500:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607094592/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233227 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233202.459506:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233202.459508:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233202.459510:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094592:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233202.459514:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094656:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233202.459529:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233202/real 1761233202] req@00000000db10af3c x1846780607094656/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233227 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233202.459537:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607094656/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233227 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233202.459544:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233202.459546:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233202.459548:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094656:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233202.459571:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094720:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233202.459587:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233202/real 1761233202] req@00000000d07717c4 x1846780607094720/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233227 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233202.459595:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607094720/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233227 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233202.459602:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233202.459603:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233202.459606:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094720:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233202.907034:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233203.931023:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233204.954984:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233205.979071:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233207.002979:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233207.578976:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233207.578985:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233207.578988:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233207.578997:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6079 00000100:00080000:0.0:1761233207.579000:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233207.579006:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233207.579037:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607094784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233207.579058:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233207.579060:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233207.579061:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233207.579064:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6079 00000100:00080000:0.0:1761233207.579065:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233207.579067:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233207.579074:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233207.579082:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607094848/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233207.579087:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233207.579089:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233207.579090:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233207.579097:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6079 00000100:00080000:0.0:1761233207.579098:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233207.579100:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233207.579103:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233207.579108:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607094912/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233207.579114:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233207.579116:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233207.579118:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233207.579120:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6079 00000100:00080000:0.0:1761233207.579122:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233207.579124:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233207.579126:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233207.579132:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607094976/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233207.579137:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607094784:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233207.579226:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233207/real 1761233207] req@000000007f60e210 x1846780607094784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233232 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233207.579239:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607094784/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233232 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233207.579247:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233207.579249:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233207.579251:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607094784:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233207.579256:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094848:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233207.579272:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233207/real 1761233207] req@00000000fcce404c x1846780607094848/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233232 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233207.579278:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607094848/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233232 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233207.579283:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233207.579284:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233207.579285:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094848:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233207.579289:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094912:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233207.579300:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233207/real 1761233207] req@00000000b8f2be78 x1846780607094912/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233232 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233207.579306:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607094912/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233232 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233207.579311:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233207.579312:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233207.579313:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094912:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233207.579332:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094976:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233207.579344:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233207/real 1761233207] req@000000000243251f x1846780607094976/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233232 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233207.579350:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607094976/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233232 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233207.579355:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233207.579356:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233207.579358:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607094976:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233208.027072:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233209.051060:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233210.075055:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233211.099068:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233212.123000:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233212.698984:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233212.698994:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233212.698997:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233212.699006:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6085 00000100:00080000:3.0:1761233212.699009:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233212.699014:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233212.699044:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607095040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233212.699065:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233212.699068:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233212.699069:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233212.699072:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6085 00000100:00080000:3.0:1761233212.699074:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233212.699076:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233212.699083:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233212.699090:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607095104/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233212.699095:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233212.699098:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233212.699098:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233212.699100:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6085 00000100:00080000:3.0:1761233212.699101:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233212.699103:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233212.699106:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233212.699127:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607095168/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233212.699136:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233212.699139:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233212.699140:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233212.699144:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6085 00000100:00080000:3.0:1761233212.699146:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233212.699148:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233212.699151:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233212.699158:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607095232/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233212.699188:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607095040:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233212.699323:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233212/real 1761233212] req@000000006572b591 x1846780607095040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233237 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233212.699345:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607095040/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233237 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233212.699357:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233212.699362:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233212.699367:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607095040:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233212.699377:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095104:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233212.699411:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233212/real 1761233212] req@00000000fc01c41c x1846780607095104/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233237 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233212.699425:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607095104/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233237 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233212.699433:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233212.699436:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233212.699439:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095104:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233212.699447:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095168:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233212.699475:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233212/real 1761233212] req@00000000408f088c x1846780607095168/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233237 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233212.699487:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607095168/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233237 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233212.699496:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233212.699499:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233212.699502:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095168:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233212.699538:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095232:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233212.699568:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233212/real 1761233212] req@000000002ce934d3 x1846780607095232/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233237 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233212.699580:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607095232/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233237 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233212.699588:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233212.699590:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233212.699593:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095232:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233213.146980:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233214.171056:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233215.194973:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233216.219023:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233217.242969:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233217.819024:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233217.819073:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233217.819077:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233217.819088:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6090 00000100:00080000:1.0:1761233217.819093:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233217.819100:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233217.819140:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607095296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233217.819165:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233217.819169:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233217.819171:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233217.819174:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6090 00000100:00080000:1.0:1761233217.819177:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233217.819180:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233217.819188:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233217.819202:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607095360/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233217.819212:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761233217.819215:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607095296:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761233217.819217:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233217.819220:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233217.819228:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6090 00000100:00080000:1.0:1761233217.819230:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233217.819233:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233217.819239:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233217.819251:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607095424/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233217.819261:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233217.819264:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233217.819266:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233217.819269:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6090 00000100:00080000:1.0:1761233217.819272:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233217.819275:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233217.819280:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233217.819291:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607095488/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233217.819313:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233217/real 1761233217] req@00000000d07717c4 x1846780607095296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233242 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233217.819330:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607095296/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233242 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233217.819337:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233217.819340:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233217.819343:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607095296:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233217.819362:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095360:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233217.819380:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233217/real 1761233217] req@00000000db10af3c x1846780607095360/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233242 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233217.819388:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607095360/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233242 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233217.819392:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233217.819394:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233217.819395:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095360:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233217.819399:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095424:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233217.819410:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233217/real 1761233217] req@00000000659d78e3 x1846780607095424/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233242 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233217.819416:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607095424/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233242 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233217.819421:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233217.819422:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233217.819424:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095424:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233217.819427:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095488:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233217.819438:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233217/real 1761233217] req@00000000b72feabb x1846780607095488/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233242 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233217.819443:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607095488/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233242 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233217.819448:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233217.819449:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233217.819451:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095488:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233218.267001:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233219.291034:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233220.315065:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233221.339038:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233222.363014:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233222.938965:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233222.938975:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233222.938978:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233222.938987:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6095 00000100:00080000:0.0:1761233222.938990:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233222.938995:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233222.939025:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607095552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233222.939047:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233222.939049:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233222.939050:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233222.939053:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6095 00000100:00080000:0.0:1761233222.939054:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233222.939058:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233222.939064:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233222.939073:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607095616/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233222.939079:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233222.939082:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233222.939083:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233222.939085:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6095 00000100:00080000:0.0:1761233222.939086:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233222.939088:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233222.939091:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233222.939093:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607095552:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761233222.939100:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607095680/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233222.939106:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233222.939109:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233222.939110:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233222.939112:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6095 00000100:00080000:0.0:1761233222.939114:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233222.939115:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233222.939118:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233222.939124:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607095744/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233222.939167:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233222/real 1761233222] req@00000000271a9d8e x1846780607095552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233247 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233222.939180:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607095552/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233247 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233222.939187:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233222.939189:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233222.939191:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607095552:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233222.939196:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095616:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233222.939211:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233222/real 1761233222] req@0000000001b2f813 x1846780607095616/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233247 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233222.939222:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607095616/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233247 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233222.939226:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233222.939228:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233222.939229:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095616:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233222.939248:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095680:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233222.939261:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233222/real 1761233222] req@0000000023ab3a3f x1846780607095680/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233247 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233222.939268:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607095680/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233247 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233222.939273:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233222.939274:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233222.939275:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095680:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233222.939279:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095744:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233222.939290:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233222/real 1761233222] req@00000000275aeee5 x1846780607095744/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233247 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233222.939296:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607095744/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233247 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233222.939301:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233222.939302:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233222.939304:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095744:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233223.386988:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233224.411019:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233225.435008:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233226.459052:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233227.483082:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233228.058978:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233228.058988:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233228.058991:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233228.059001:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6100 00000100:00080000:0.0:1761233228.059004:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233228.059012:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233228.059055:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607095808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233228.059090:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233228.059095:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233228.059097:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233228.059102:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6100 00000100:00080000:0.0:1761233228.059105:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233228.059109:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233228.059119:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233228.059135:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607095872/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233228.059148:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761233228.059153:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607095808:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761233228.059155:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233228.059158:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233228.059168:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6100 00000100:00080000:0.0:1761233228.059171:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233228.059175:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233228.059181:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233228.059200:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607095936/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233228.059210:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233228.059214:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233228.059216:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233228.059220:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6100 00000100:00080000:0.0:1761233228.059222:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233228.059226:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233228.059231:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233228.059245:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607096000/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233228.059265:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233228/real 1761233228] req@000000002ef1f54f x1846780607095808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233253 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233228.059283:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607095808/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233253 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233228.059293:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233228.059296:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233228.059300:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607095808:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233228.059322:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095872:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233228.059343:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233228/real 1761233228] req@0000000099514377 x1846780607095872/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233253 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233228.059351:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607095872/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233253 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233228.059357:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233228.059359:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233228.059361:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095872:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233228.059365:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095936:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233228.059379:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233228/real 1761233228] req@000000005a096598 x1846780607095936/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233253 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233228.059388:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607095936/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233253 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233228.059394:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233228.059395:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233228.059397:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607095936:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233228.059401:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096000:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233228.059414:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233228/real 1761233228] req@00000000275aeee5 x1846780607096000/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233253 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233228.059421:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607096000/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233253 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233228.059427:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233228.059428:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233228.059430:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096000:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233228.507013:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233229.530995:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233230.554969:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233231.579088:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233232.603061:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233233.179051:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233233.179066:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233233.179071:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233233.179083:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6105 00000100:00080000:1.0:1761233233.179088:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233233.179097:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233233.179144:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607096064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233233.179180:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233233.179185:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233233.179187:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233233.179192:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6105 00000100:00080000:1.0:1761233233.179194:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233233.179198:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233233.179214:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233233.179234:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607096128/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233233.179252:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233233.179258:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233233.179260:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233233.179268:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6105 00000100:00080000:1.0:1761233233.179273:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233233.179280:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233233.179290:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233233.179312:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607096192/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233233.179319:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607096064:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761233233.179346:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233233.179352:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233233.179354:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233233.179362:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6105 00000100:00080000:1.0:1761233233.179364:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233233.179370:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233233.179388:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233233.179404:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607096256/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233233.179448:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233233/real 1761233233] req@00000000b72feabb x1846780607096064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233258 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233233.179474:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607096064/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233258 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233233.179494:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233233.179498:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233233.179503:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607096064:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233233.179512:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096128:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233233.179540:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233233/real 1761233233] req@00000000659d78e3 x1846780607096128/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233258 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233233.179553:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607096128/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233258 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233233.179561:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233233.179564:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233233.179566:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096128:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233233.179599:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096192:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233233.179644:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233233/real 1761233233] req@00000000db10af3c x1846780607096192/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233258 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233233.179668:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607096192/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233258 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233233.179683:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233233.179687:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233233.179692:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096192:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233233.179702:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096256:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233233.179748:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233233/real 1761233233] req@00000000d07717c4 x1846780607096256/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233258 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233233.179766:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607096256/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233258 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233233.179780:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233233.179784:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233233.179788:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096256:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233233.627019:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233234.650973:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233235.675004:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233236.699091:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233237.723164:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233238.298987:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233238.298999:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233238.299004:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233238.299017:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6110 00000100:00080000:0.0:1761233238.299020:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233238.299027:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233238.299071:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607096320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233238.299096:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233238.299099:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233238.299101:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233238.299104:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6110 00000100:00080000:0.0:1761233238.299106:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233238.299108:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233238.299116:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233238.299127:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607096384/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233238.299133:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233238.299135:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233238.299136:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233238.299140:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6110 00000100:00080000:0.0:1761233238.299141:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233238.299143:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233238.299146:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233238.299153:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607096448/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233238.299158:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233238.299161:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233238.299162:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233238.299164:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6110 00000100:00080000:0.0:1761233238.299166:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233238.299167:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233238.299170:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233238.299175:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607096512/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233238.299200:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607096320:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233238.299298:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233238/real 1761233238] req@0000000023ab3a3f x1846780607096320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233263 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233238.299315:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607096320/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233263 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233238.299325:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233238.299329:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233238.299332:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607096320:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233238.299340:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096384:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233238.299365:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233238/real 1761233238] req@0000000001b2f813 x1846780607096384/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233263 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233238.299375:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607096384/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233263 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233238.299383:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233238.299385:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233238.299388:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096384:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233238.299394:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096448:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233238.299414:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233238/real 1761233238] req@00000000271a9d8e x1846780607096448/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233263 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233238.299423:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607096448/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233263 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233238.299431:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233238.299433:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233238.299435:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096448:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233238.299461:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096512:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233238.299483:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233238/real 1761233238] req@000000000243251f x1846780607096512/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233263 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233238.299493:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607096512/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233263 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233238.299500:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233238.299502:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233238.299505:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096512:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:0.0:1761233238.747037:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233239.771038:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233240.795060:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233241.819025:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233242.842998:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233243.418995:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233243.419007:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233243.419011:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233243.419022:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6115 00000100:00080000:1.0:1761233243.419026:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233243.419033:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233243.419072:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607096576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233243.419100:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233243.419104:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233243.419106:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233243.419110:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6115 00000100:00080000:1.0:1761233243.419112:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233243.419116:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233243.419125:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233243.419137:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607096640/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233243.419147:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233243.419150:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233243.419152:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233243.419157:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6115 00000100:00080000:1.0:1761233243.419160:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233243.419164:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233243.419169:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233243.419178:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607096576:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233243.419183:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607096704/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233243.419195:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233243.419199:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233243.419201:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233243.419204:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6115 00000100:00080000:1.0:1761233243.419206:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233243.419210:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233243.419214:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233243.419225:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607096768/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233243.419256:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233243/real 1761233243] req@00000000d07717c4 x1846780607096576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233268 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233243.419270:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607096576/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233268 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233243.419277:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233243.419280:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233243.419282:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607096576:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233243.419287:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096640:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233243.419301:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233243/real 1761233243] req@00000000db10af3c x1846780607096640/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233268 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233243.419307:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607096640/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233268 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233243.419312:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233243.419313:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233243.419314:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096640:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233243.419332:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096704:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233243.419345:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233243/real 1761233243] req@00000000659d78e3 x1846780607096704/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233268 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233243.419351:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607096704/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233268 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233243.419356:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233243.419358:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233243.419359:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096704:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233243.419362:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096768:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233243.419372:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233243/real 1761233243] req@00000000b72feabb x1846780607096768/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233268 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233243.419377:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607096768/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233268 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233243.419382:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233243.419384:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233243.419385:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096768:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233243.867035:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233244.890975:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233245.915112:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233246.939113:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233247.963037:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233248.539005:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233248.539018:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233248.539024:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233248.539036:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6120 00000100:00080000:0.0:1761233248.539040:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233248.539048:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233248.539092:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607096832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233248.539125:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233248.539129:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233248.539131:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233248.539135:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6120 00000100:00080000:0.0:1761233248.539138:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233248.539141:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233248.539165:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233248.539179:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607096896/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233248.539189:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233248.539194:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233248.539196:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233248.539200:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6120 00000100:00080000:0.0:1761233248.539204:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233248.539208:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233248.539213:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233248.539224:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607096960/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233248.539232:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607096832:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761233248.539238:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233248.539247:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233248.539249:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233248.539253:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6120 00000100:00080000:0.0:1761233248.539256:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233248.539258:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233248.539262:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233248.539274:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607097024/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233248.539361:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233248/real 1761233248] req@00000000b8f2be78 x1846780607096832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233273 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233248.539389:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607096832/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233273 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233248.539404:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233248.539409:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233248.539415:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607096832:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233248.539427:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096896:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233248.539466:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233248/real 1761233248] req@00000000fcce404c x1846780607096896/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233273 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233248.539484:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607096896/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233273 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233248.539496:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233248.539499:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233248.539503:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096896:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233248.539536:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096960:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233248.539604:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233248/real 1761233248] req@000000007f60e210 x1846780607096960/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233273 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233248.539622:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607096960/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233273 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233248.539636:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233248.539639:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233248.539644:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607096960:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233248.539653:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097024:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233248.539687:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233248/real 1761233248] req@000000000243251f x1846780607097024/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233273 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233248.539704:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607097024/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233273 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233248.539717:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233248.539720:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233248.539725:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097024:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233248.987035:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233250.010992:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233251.035011:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233252.058992:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233253.083048:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233253.658971:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233253.658986:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233253.658991:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233253.659005:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6126 00000100:00080000:0.0:1761233253.659011:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233253.659020:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233253.659066:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607097088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233253.659107:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233253.659115:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233253.659117:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233253.659123:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6126 00000100:00080000:0.0:1761233253.659126:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233253.659132:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233253.659145:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233253.659165:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607097152/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233253.659173:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607097088:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761233253.659217:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233253.659221:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233253.659223:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233253.659228:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6126 00000100:00080000:0.0:1761233253.659231:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233253.659235:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233253.659240:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233253.659253:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607097216/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233253.659262:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233253.659266:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233253.659267:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233253.659271:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6126 00000100:00080000:0.0:1761233253.659273:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233253.659276:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233253.659280:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233253.659288:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097152:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:0.0:1761233253.659293:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607097280/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233253.659311:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097216:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233253.659327:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097280:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233254.107053:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233255.131054:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233256.155101:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761233256.731499:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233253/real 1761233256] req@00000000275aeee5 x1846780607097280/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233278 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233256.731539:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607097280/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233278 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233256.731557:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233256.731563:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233256.731568:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097280:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233256.731578:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233253/real 1761233256] req@0000000001b2f813 x1846780607097152/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233278 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233256.731594:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607097152/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233278 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233256.731623:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233256.731626:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233256.731629:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097152:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233256.731636:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233253/real 1761233256] req@0000000023ab3a3f x1846780607097216/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233278 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233256.731647:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607097216/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233278 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233256.731656:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233256.731658:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233256.731661:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097216:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233256.731669:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233253/real 1761233256] req@00000000271a9d8e x1846780607097088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233278 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233256.731681:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607097088/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233278 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233256.731690:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233256.731693:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233256.731696:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607097088:192.168.201.104@tcp:250:kworker/u8:2.0 00010000:00010000:2.0:1761233257.179189:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233258.203071:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233258.779057:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233258.779070:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233258.779075:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233258.779088:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6131 00000100:00080000:3.0:1761233258.779093:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233258.779100:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233258.779143:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607097344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233258.779183:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233258.779187:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233258.779189:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233258.779194:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6131 00000100:00080000:3.0:1761233258.779196:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233258.779215:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233258.779230:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233258.779260:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607097408/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233258.779276:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233258.779283:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233258.779286:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233258.779293:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6131 00000100:00080000:3.0:1761233258.779297:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233258.779304:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233258.779313:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233258.779334:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607097472/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233258.779340:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607097344:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761233258.779369:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233258.779375:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233258.779379:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233258.779421:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6131 00000100:00080000:3.0:1761233258.779426:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233258.779433:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233258.779444:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233258.779467:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607097536/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233258.779530:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233258/real 1761233258] req@000000002ce934d3 x1846780607097344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233283 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233258.779566:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607097344/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233283 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233258.779584:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233258.779590:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233258.779597:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607097344:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233258.779609:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097408:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233258.779663:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233258/real 1761233258] req@00000000408f088c x1846780607097408/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233283 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233258.779685:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607097408/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233283 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233258.779700:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233258.779704:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233258.779710:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097408:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233258.779752:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097472:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233258.779805:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233258/real 1761233258] req@00000000fc01c41c x1846780607097472/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233283 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233258.779841:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607097472/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233283 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233258.779930:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233258.779933:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233258.779938:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097472:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233258.779948:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097536:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233258.779988:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233258/real 1761233258] req@000000006572b591 x1846780607097536/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233283 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233258.780004:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607097536/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233283 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233258.780015:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233258.780018:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233258.780022:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097536:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233259.227082:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233260.251050:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233261.275148:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233262.299019:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233263.323073:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233263.898955:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233263.898970:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233263.898976:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233263.898988:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6136 00000100:00080000:3.0:1761233263.898992:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233263.899000:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233263.899051:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607097600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233263.899074:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233263.899078:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233263.899079:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233263.899082:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6136 00000100:00080000:3.0:1761233263.899084:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233263.899086:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233263.899098:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233263.899110:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607097664/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233263.899116:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233263.899119:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233263.899120:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233263.899123:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6136 00000100:00080000:3.0:1761233263.899125:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233263.899129:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233263.899134:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233263.899146:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607097728/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233263.899152:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233263.899154:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233263.899155:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233263.899158:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6136 00000100:00080000:3.0:1761233263.899160:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1761233263.899168:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607097600:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761233263.899169:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233263.899177:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233263.899184:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607097792/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233263.899283:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233263/real 1761233263] req@000000001f8e688e x1846780607097600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233288 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233263.899299:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607097600/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233288 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233263.899322:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233263.899325:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233263.899329:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607097600:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233263.899334:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097664:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233263.899354:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233263/real 1761233263] req@00000000a58ca11e x1846780607097664/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233288 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233263.899366:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607097664/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233288 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233263.899371:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233263.899373:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233263.899375:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097664:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233263.899378:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097728:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233263.899390:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233263/real 1761233263] req@0000000083f16a6b x1846780607097728/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233288 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233263.899396:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607097728/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233288 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233263.899402:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233263.899404:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233263.899405:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097728:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233263.899432:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097792:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233263.899444:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233263/real 1761233263] req@0000000090b40c66 x1846780607097792/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233288 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233263.899451:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607097792/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233288 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233263.899456:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233263.899457:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233263.899460:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097792:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:0.0:1761233264.347123:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761233265.371074:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233266.395039:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233267.419145:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233268.443201:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233269.019004:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233269.019019:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233269.019024:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233269.019037:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6141 00000100:00080000:2.0:1761233269.019041:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233269.019048:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233269.019092:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607097856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233269.019119:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233269.019122:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233269.019124:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233269.019127:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6141 00000100:00080000:2.0:1761233269.019128:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233269.019131:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233269.019139:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233269.019147:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607097920/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233269.019153:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233269.019155:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233269.019156:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233269.019159:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6141 00000100:00080000:2.0:1761233269.019162:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233269.019165:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233269.019168:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233269.019175:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607097984/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233269.019180:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233269.019182:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233269.019184:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233269.019187:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6141 00000100:00080000:2.0:1761233269.019189:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233269.019191:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233269.019195:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233269.019203:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607098048/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233269.019251:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607097856:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233269.019344:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233269/real 1761233269] req@0000000091106a97 x1846780607097856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233294 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233269.019363:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607097856/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233294 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233269.019377:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233269.019380:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233269.019385:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607097856:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233269.019393:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097920:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233269.019420:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233269/real 1761233269] req@0000000094135f44 x1846780607097920/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233294 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233269.019431:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607097920/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233294 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233269.019438:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233269.019442:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233269.019445:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097920:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233269.019451:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097984:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233269.019475:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233269/real 1761233269] req@0000000064978143 x1846780607097984/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233294 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233269.019485:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607097984/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233294 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233269.019492:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233269.019494:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233269.019497:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607097984:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233269.019502:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098048:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233269.019523:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233269/real 1761233269] req@00000000c069b612 x1846780607098048/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233294 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233269.019532:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607098048/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233294 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233269.019539:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233269.019541:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233269.019544:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098048:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761233269.467079:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233270.491159:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233271.515079:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233272.539065:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233273.563036:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233274.139059:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233274.139071:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233274.139076:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233274.139088:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6146 00000100:00080000:0.0:1761233274.139092:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233274.139101:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233274.139187:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607098112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233274.139230:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233274.139236:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233274.139239:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233274.139245:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6146 00000100:00080000:0.0:1761233274.139249:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233274.139253:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233274.139277:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233274.139296:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607098176/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233274.139306:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233274.139310:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233274.139313:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233274.139316:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6146 00000100:00080000:0.0:1761233274.139318:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233274.139322:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233274.139327:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233274.139338:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607098240/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233274.139348:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607098112:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761233274.139351:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233274.139361:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233274.139362:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233274.139367:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6146 00000100:00080000:0.0:1761233274.139369:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233274.139372:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233274.139377:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233274.139388:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607098304/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233274.139456:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233274/real 1761233274] req@000000005a096598 x1846780607098112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233299 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233274.139480:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607098112/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233299 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233274.139491:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233274.139494:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233274.139498:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607098112:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233274.139504:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098176:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233274.139525:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233274/real 1761233274] req@0000000099514377 x1846780607098176/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233299 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233274.139534:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607098176/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233299 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233274.139541:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233274.139543:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233274.139546:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098176:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233274.139569:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098240:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233274.139588:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233274/real 1761233274] req@000000002ef1f54f x1846780607098240/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233299 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233274.139597:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607098240/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233299 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233274.139604:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233274.139606:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233274.139609:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098240:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233274.139614:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098304:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233274.139631:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233274/real 1761233274] req@00000000271a9d8e x1846780607098304/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233299 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233274.139640:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607098304/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233299 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233274.139647:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233274.139649:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233274.139651:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098304:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761233274.587041:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233275.611103:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233276.635044:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233277.659045:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233278.683110:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233279.258974:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233279.258985:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233279.258988:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233279.258998:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6151 00000100:00080000:0.0:1761233279.259001:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233279.259007:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233279.259038:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607098368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233279.259063:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233279.259088:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233279.259090:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233279.259093:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6151 00000100:00080000:0.0:1761233279.259094:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233279.259097:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233279.259105:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233279.259131:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607098432/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233279.259137:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233279.259139:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233279.259140:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233279.259142:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6151 00000100:00080000:0.0:1761233279.259144:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233279.259146:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233279.259149:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233279.259156:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607098496/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233279.259162:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233279.259164:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233279.259165:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233279.259167:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6151 00000100:00080000:0.0:1761233279.259168:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1761233279.259171:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607098368:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761233279.259173:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233279.259179:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233279.259186:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607098560/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233279.259249:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233279/real 1761233279] req@0000000023ab3a3f x1846780607098368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233304 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233279.259261:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607098368/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233304 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233279.259268:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233279.259270:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233279.259273:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607098368:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233279.259278:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098432:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233279.259292:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233279/real 1761233279] req@0000000001b2f813 x1846780607098432/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233304 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233279.259298:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607098432/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233304 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233279.259307:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233279.259308:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233279.259310:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098432:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233279.259328:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098496:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233279.259340:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233279/real 1761233279] req@00000000275aeee5 x1846780607098496/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233304 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233279.259346:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607098496/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233304 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233279.259351:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233279.259352:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233279.259354:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098496:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233279.259357:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098560:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233279.259367:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233279/real 1761233279] req@000000000243251f x1846780607098560/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233304 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233279.259373:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607098560/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233304 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233279.259378:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233279.259379:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233279.259380:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098560:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761233279.707125:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233280.731120:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233281.755008:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233282.779070:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233283.803012:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233284.379055:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233284.379066:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233284.379070:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233284.379080:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6156 00000100:00080000:1.0:1761233284.379083:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233284.379089:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233284.379126:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607098624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233284.379153:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233284.379156:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233284.379157:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233284.379161:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6156 00000100:00080000:1.0:1761233284.379162:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233284.379164:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233284.379171:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233284.379180:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607098688/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233284.379195:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233284.379198:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233284.379201:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233284.379204:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6156 00000100:00080000:1.0:1761233284.379205:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233284.379207:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233284.379211:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233284.379219:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607098752/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233284.379224:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607098624:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:1.0:1761233284.379231:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233284.379236:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233284.379237:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233284.379240:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6156 00000100:00080000:1.0:1761233284.379241:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233284.379244:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233284.379248:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233284.379258:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607098816/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233284.379314:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233284/real 1761233284] req@00000000b72feabb x1846780607098624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233309 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233284.379335:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607098624/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233309 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233284.379343:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233284.379345:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233284.379348:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607098624:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233284.379354:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098688:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233284.379373:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233284/real 1761233284] req@00000000659d78e3 x1846780607098688/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233309 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233284.379381:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607098688/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233309 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233284.379386:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233284.379387:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233284.379389:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098688:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233284.379410:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098752:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233284.379426:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233284/real 1761233284] req@00000000db10af3c x1846780607098752/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233309 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233284.379434:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607098752/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233309 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233284.379440:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233284.379441:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233284.379444:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098752:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233284.379448:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098816:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233284.379462:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233284/real 1761233284] req@00000000d07717c4 x1846780607098816/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233309 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233284.379470:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607098816/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233309 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233284.379475:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233284.379477:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233284.379478:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098816:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761233284.827049:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233285.851044:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233286.875073:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233287.899078:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233288.923024:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233289.498974:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233289.498984:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233289.498988:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233289.498998:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6161 00000100:00080000:2.0:1761233289.499002:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233289.499008:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233289.499044:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607098880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233289.499072:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233289.499075:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233289.499077:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233289.499081:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6161 00000100:00080000:2.0:1761233289.499083:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233289.499087:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233289.499095:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233289.499106:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607098944/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233289.499114:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233289.499117:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233289.499118:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233289.499125:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6161 00000100:00100000:1.0:1761233289.499126:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607098880:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:2.0:1761233289.499127:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233289.499130:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233289.499137:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233289.499150:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607099008/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233289.499157:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233289.499160:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233289.499162:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233289.499165:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6161 00000100:00080000:2.0:1761233289.499167:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233289.499169:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233289.499172:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233289.499180:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607099072/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233289.499220:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233289/real 1761233289] req@00000000f38f27df x1846780607098880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233314 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233289.499235:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607098880/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233314 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233289.499243:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233289.499246:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233289.499249:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607098880:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233289.499269:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098944:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233289.499291:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233289/real 1761233289] req@0000000017386078 x1846780607098944/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233314 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233289.499300:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607098944/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233314 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233289.499307:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233289.499309:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233289.499311:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607098944:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233289.499316:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099008:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233289.499330:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233289/real 1761233289] req@00000000c069b612 x1846780607099008/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233314 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233289.499338:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607099008/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233314 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233289.499344:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233289.499346:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233289.499348:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099008:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233289.499352:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099072:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233289.499366:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233289/real 1761233289] req@0000000064978143 x1846780607099072/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233314 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233289.499374:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607099072/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233314 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233289.499380:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233289.499381:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233289.499383:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099072:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761233289.947192:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233290.971082:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233291.995075:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233293.019040:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233294.043034:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233294.619000:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233294.619018:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233294.619024:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233294.619039:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6167 00000100:00080000:0.0:1761233294.619045:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233294.619055:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233294.619108:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607099136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233294.619163:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233294.619170:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233294.619173:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233294.619181:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6167 00000100:00080000:0.0:1761233294.619185:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233294.619191:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233294.619205:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233294.619230:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607099200/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233294.619244:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607099136:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761233294.619256:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233294.619265:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233294.619268:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233294.619280:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6167 00000100:00080000:0.0:1761233294.619285:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233294.619291:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233294.619300:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233294.619324:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607099264/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233294.619341:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233294.619347:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233294.619350:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233294.619357:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6167 00000100:00080000:0.0:1761233294.619361:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233294.619369:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1761233294.619373:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233294/real 1761233294] req@000000007f60e210 x1846780607099136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233319 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000008:00100000:0.0:1761233294.619380:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233294.619406:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607099136/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233319 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:0.0:1761233294.619413:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607099328/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233294.619433:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233294.619443:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233294.619452:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607099136:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233294.619483:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099200:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233294.619515:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233294/real 1761233294] req@00000000fcce404c x1846780607099200/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233319 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233294.619527:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607099200/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233319 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233294.619536:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233294.619539:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233294.619542:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099200:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233294.619549:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099264:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233294.619571:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233294/real 1761233294] req@00000000b8f2be78 x1846780607099264/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233319 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233294.619582:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607099264/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233319 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233294.619592:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233294.619595:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233294.619598:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099264:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233294.619604:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099328:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233294.619629:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233294/real 1761233294] req@000000000243251f x1846780607099328/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233319 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233294.619648:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607099328/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233319 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233294.619663:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233294.619667:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233294.619672:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099328:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761233295.067067:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233296.091115:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233297.115033:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233298.139001:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233299.163050:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233299.739005:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233299.739013:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233299.739016:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233299.739025:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6172 00000100:00080000:2.0:1761233299.739028:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233299.739033:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233299.739063:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607099392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233299.739086:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233299.739089:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233299.739091:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233299.739095:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6172 00000100:00080000:2.0:1761233299.739107:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233299.739110:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233299.739119:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233299.739134:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607099456/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233299.739142:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607099392:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761233299.739146:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233299.739151:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233299.739153:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233299.739157:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6172 00000100:00080000:2.0:1761233299.739160:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233299.739163:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233299.739167:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233299.739178:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607099520/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233299.739184:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233299.739186:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233299.739187:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233299.739188:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6172 00000100:00080000:2.0:1761233299.739190:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233299.739192:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233299.739195:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233299.739201:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607099584/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233299.739222:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233299/real 1761233299] req@0000000094135f44 x1846780607099392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233324 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233299.739236:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607099392/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233324 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233299.739243:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233299.739246:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233299.739248:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607099392:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233299.739266:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099456:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233299.739282:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233299/real 1761233299] req@0000000091106a97 x1846780607099456/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233324 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233299.739294:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607099456/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233324 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233299.739298:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233299.739300:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233299.739301:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099456:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233299.739304:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099520:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233299.739316:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233299/real 1761233299] req@0000000064978143 x1846780607099520/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233324 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233299.739322:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607099520/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233324 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233299.739327:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233299.739328:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233299.739329:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099520:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233299.739332:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099584:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233299.739343:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233299/real 1761233299] req@00000000c069b612 x1846780607099584/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233324 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233299.739348:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607099584/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233324 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233299.739353:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233299.739354:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233299.739356:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099584:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761233300.186992:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233301.211034:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233302.235189:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233303.259008:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233304.282997:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233304.858897:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233304.858922:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233304.858926:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233304.858936:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6177 00000100:00080000:2.0:1761233304.858939:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233304.858946:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233304.858983:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607099648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233304.859009:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233304.859013:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233304.859014:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233304.859018:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6177 00000100:00080000:2.0:1761233304.859020:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233304.859022:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233304.859029:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233304.859038:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607099712/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233304.859045:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233304.859048:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233304.859049:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233304.859052:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6177 00000100:00080000:2.0:1761233304.859053:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233304.859055:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233304.859058:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233304.859065:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607099776/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233304.859071:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233304.859073:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233304.859074:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233304.859077:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6177 00000100:00080000:2.0:1761233304.859079:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233304.859082:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233304.859085:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233304.859093:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607099840/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233304.859116:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607099648:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233304.859217:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233304/real 1761233304] req@0000000017386078 x1846780607099648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233329 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233304.859253:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607099648/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233329 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233304.859263:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233304.859266:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233304.859270:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607099648:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233304.859277:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099712:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233304.859301:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233304/real 1761233304] req@00000000f38f27df x1846780607099712/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233329 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233304.859308:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607099712/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233329 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233304.859313:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233304.859315:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233304.859316:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099712:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233304.859319:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099776:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233304.859331:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233304/real 1761233304] req@00000000c069b612 x1846780607099776/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233329 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233304.859337:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607099776/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233329 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233304.859343:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233304.859345:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233304.859347:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099776:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233304.859350:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099840:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233304.859360:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233304/real 1761233304] req@0000000064978143 x1846780607099840/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233329 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233304.859365:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607099840/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233329 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233304.859371:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233304.859372:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233304.859374:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099840:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761233305.307045:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233306.331002:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233307.355010:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233308.379135:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233309.402992:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233309.978948:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233309.978958:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233309.978962:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233309.978973:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6182 00000100:00080000:1.0:1761233309.978976:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233309.978984:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233309.979022:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607099904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233309.979050:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233309.979053:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233309.979055:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233309.979060:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6182 00000100:00080000:1.0:1761233309.979062:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233309.979065:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233309.979074:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233309.979084:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607099968/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233309.979095:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761233309.979098:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607099904:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761233309.979100:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233309.979102:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233309.979109:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6182 00000100:00080000:1.0:1761233309.979112:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233309.979115:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233309.979119:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233309.979131:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607100032/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233309.979139:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233309.979142:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233309.979144:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233309.979147:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6182 00000100:00080000:1.0:1761233309.979149:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233309.979153:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233309.979157:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233309.979167:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607100096/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233309.979199:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233309/real 1761233309] req@00000000d07717c4 x1846780607099904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233334 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233309.979219:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607099904/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233334 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233309.979229:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233309.979232:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233309.979236:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607099904:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233309.979260:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099968:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233309.979290:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233309/real 1761233309] req@00000000db10af3c x1846780607099968/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233334 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233309.979302:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607099968/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233334 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233309.979308:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233309.979311:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233309.979314:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607099968:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233309.979319:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100032:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233309.979348:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233309/real 1761233309] req@00000000659d78e3 x1846780607100032/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233334 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233309.979359:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607100032/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233334 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233309.979367:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233309.979369:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233309.979372:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100032:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233309.979379:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100096:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233309.979403:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233309/real 1761233309] req@00000000b72feabb x1846780607100096/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233334 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233309.979415:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607100096/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233334 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233309.979423:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233309.979425:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233309.979428:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100096:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233310.427057:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233311.451083:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233312.474991:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233313.498991:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233314.523188:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233315.098982:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233315.098995:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233315.098999:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233315.099009:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6187 00000100:00080000:1.0:1761233315.099013:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233315.099020:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233315.099057:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607100160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233315.099089:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233315.099094:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233315.099095:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233315.099100:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6187 00000100:00080000:1.0:1761233315.099102:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233315.099106:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233315.099114:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233315.099127:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607100224/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233315.099136:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233315.099139:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233315.099141:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233315.099144:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6187 00000100:00080000:1.0:1761233315.099146:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233315.099149:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233315.099156:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233315.099167:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607100288/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233315.099176:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233315.099180:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233315.099181:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761233315.099184:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607100160:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761233315.099188:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6187 00000100:00080000:1.0:1761233315.099193:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233315.099197:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233315.099201:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233315.099214:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607100352/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233315.099265:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233315/real 1761233315] req@00000000b72feabb x1846780607100160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233340 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233315.099278:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607100160/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233340 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233315.099286:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233315.099288:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233315.099291:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607100160:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233315.099295:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100224:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233315.099309:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233315/real 1761233315] req@00000000659d78e3 x1846780607100224/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233340 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233315.099316:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607100224/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233340 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233315.099320:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233315.099321:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233315.099323:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100224:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233315.099340:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100288:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233315.099352:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233315/real 1761233315] req@00000000db10af3c x1846780607100288/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233340 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233315.099358:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607100288/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233340 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233315.099363:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233315.099364:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233315.099365:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100288:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233315.099368:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100352:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233315.099378:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233315/real 1761233315] req@00000000d07717c4 x1846780607100352/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233340 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233315.099384:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607100352/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233340 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233315.099389:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233315.099390:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233315.099392:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100352:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233315.547045:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233316.570998:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233317.595074:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233318.619137:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233319.643152:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233320.218975:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233320.218987:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233320.218992:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233320.219005:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6192 00000100:00080000:1.0:1761233320.219010:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233320.219018:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233320.219064:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607100416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233320.219097:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233320.219102:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233320.219104:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233320.219108:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6192 00000100:00080000:1.0:1761233320.219112:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233320.219116:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233320.219126:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233320.219139:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607100480/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233320.219149:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233320.219153:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233320.219154:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233320.219158:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6192 00000100:00080000:1.0:1761233320.219159:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233320.219163:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233320.219167:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233320.219177:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607100544/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233320.219188:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233320.219192:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233320.219193:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233320.219197:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6192 00000100:00080000:1.0:1761233320.219200:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233320.219204:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233320.219209:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233320.219224:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607100416:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233320.219225:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607100608/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233320.219345:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100480:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233320.219372:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100544:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233320.219402:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100608:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233320.667174:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233321.691139:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233322.715154:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761233323.291513:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233320/real 1761233323] req@00000000b72feabb x1846780607100608/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233345 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233323.291587:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607100608/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233345 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233323.291620:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233323.291638:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233323.291646:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100608:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233323.291663:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233320/real 1761233323] req@00000000d07717c4 x1846780607100416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233345 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233323.291686:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607100416/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233345 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233323.291699:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233323.291704:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233323.291709:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607100416:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233323.291720:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233320/real 1761233323] req@00000000db10af3c x1846780607100480/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233345 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233323.291739:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607100480/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233345 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233323.291753:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233323.291758:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233323.291764:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100480:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233323.291775:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233320/real 1761233323] req@00000000659d78e3 x1846780607100544/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233345 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233323.291795:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607100544/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233345 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233323.291811:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233323.291817:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233323.291822:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100544:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761233323.739093:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233324.763093:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233325.338970:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233325.338978:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233325.338981:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233325.338990:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6197 00000100:00080000:0.0:1761233325.338993:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233325.338998:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233325.339026:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607100672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233325.339047:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233325.339050:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233325.339051:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233325.339054:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6197 00000100:00080000:0.0:1761233325.339055:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233325.339057:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233325.339063:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233325.339072:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607100736/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233325.339078:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233325.339080:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233325.339081:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233325.339083:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6197 00000100:00080000:0.0:1761233325.339085:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233325.339086:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233325.339090:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233325.339093:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607100672:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761233325.339098:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607100800/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233325.339109:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233325.339112:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233325.339113:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233325.339116:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6197 00000100:00080000:0.0:1761233325.339118:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233325.339121:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233325.339125:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233325.339136:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607100864/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233325.339188:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233325/real 1761233325] req@00000000275aeee5 x1846780607100672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233350 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233325.339206:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607100672/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233350 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233325.339216:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233325.339219:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233325.339223:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607100672:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233325.339230:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100736:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233325.339261:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233325/real 1761233325] req@0000000001b2f813 x1846780607100736/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233350 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233325.339273:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607100736/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233350 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233325.339280:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233325.339282:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233325.339285:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100736:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233325.339311:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100800:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233325.339357:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233325/real 1761233325] req@0000000023ab3a3f x1846780607100800/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233350 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233325.339370:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607100800/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233350 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233325.339383:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233325.339386:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233325.339389:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100800:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233325.339401:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100864:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233325.339437:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233325/real 1761233325] req@00000000271a9d8e x1846780607100864/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233350 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233325.339448:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607100864/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233350 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233325.339456:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233325.339457:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233325.339460:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100864:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233325.786986:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233326.811048:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233327.835070:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233328.859064:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233329.882989:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233330.459001:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233330.459010:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233330.459013:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233330.459022:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6202 00000100:00080000:2.0:1761233330.459025:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233330.459032:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233330.459064:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607100928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233330.459088:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233330.459094:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233330.459096:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233330.459100:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6202 00000100:00080000:2.0:1761233330.459103:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233330.459108:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233330.459117:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233330.459126:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607100928:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233330.459133:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607100992/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233330.459145:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233330.459149:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233330.459151:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233330.459154:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6202 00000100:00080000:2.0:1761233330.459157:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233330.459160:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233330.459165:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233330.459178:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607101056/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233330.459186:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233330.459189:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233330.459191:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233330.459194:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6202 00000100:00080000:2.0:1761233330.459196:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233330.459200:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233330.459215:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233330.459230:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233330/real 1761233330] req@0000000091106a97 x1846780607100928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233355 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233330.459231:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607101120/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233330.459289:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607100928/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233355 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233330.459311:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233330.459314:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233330.459318:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607100928:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233330.459341:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100992:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233330.459367:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233330/real 1761233330] req@0000000094135f44 x1846780607100992/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233355 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233330.459377:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607100992/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233355 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233330.459383:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233330.459385:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233330.459388:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607100992:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233330.459392:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101056:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233330.459424:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233330/real 1761233330] req@0000000064978143 x1846780607101056/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233355 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233330.459442:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607101056/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233355 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233330.459451:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233330.459454:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233330.459457:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101056:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233330.459462:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101120:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233330.459488:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233330/real 1761233330] req@00000000c069b612 x1846780607101120/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233355 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233330.459499:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607101120/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233355 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233330.459506:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233330.459508:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233330.459511:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101120:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233330.907021:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233331.930996:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233332.955144:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233333.979058:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233335.003171:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233335.578947:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233335.578955:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233335.578958:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233335.578969:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6207 00000100:00080000:2.0:1761233335.578973:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233335.578979:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233335.579018:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607101184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233335.579046:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233335.579050:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233335.579052:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233335.579056:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6207 00000100:00080000:2.0:1761233335.579058:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233335.579062:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233335.579071:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233335.579084:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607101248/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233335.579090:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233335.579092:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233335.579093:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233335.579096:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6207 00000100:00080000:2.0:1761233335.579097:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233335.579100:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233335.579104:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233335.579111:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607101312/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233335.579116:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233335.579118:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233335.579119:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233335.579121:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6207 00000100:00080000:2.0:1761233335.579122:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233335.579124:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233335.579127:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233335.579132:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607101184:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233335.579135:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607101376/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233335.579235:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233335/real 1761233335] req@00000000f38f27df x1846780607101184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233360 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233335.579247:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607101184/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233360 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233335.579254:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233335.579256:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233335.579259:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607101184:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233335.579264:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101248:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233335.579284:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233335/real 1761233335] req@0000000017386078 x1846780607101248/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233360 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233335.579301:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607101248/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233360 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233335.579308:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233335.579310:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233335.579313:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101248:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233335.579318:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101312:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233335.579342:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233335/real 1761233335] req@00000000c069b612 x1846780607101312/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233360 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233335.579352:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607101312/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233360 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233335.579359:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233335.579361:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233335.579364:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101312:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233335.579389:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101376:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233335.579412:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233335/real 1761233335] req@0000000064978143 x1846780607101376/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233360 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233335.579421:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607101376/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233360 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233335.579427:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233335.579429:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233335.579432:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101376:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233336.027018:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233337.051007:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233338.074988:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233339.099014:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233340.123001:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233340.699001:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233340.699018:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233340.699022:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233340.699038:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6213 00000100:00080000:0.0:1761233340.699042:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233340.699052:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233340.699117:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607101440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233340.699154:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233340.699159:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233340.699161:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233340.699165:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6213 00000100:00080000:0.0:1761233340.699168:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233340.699172:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233340.699183:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233340.699200:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607101504/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233340.699213:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233340.699217:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233340.699219:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761233340.699225:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607101440:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761233340.699227:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6213 00000100:00080000:0.0:1761233340.699232:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233340.699242:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233340.699251:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233340.699267:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607101568/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233340.699277:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233340.699280:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233340.699282:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233340.699285:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6213 00000100:00080000:0.0:1761233340.699288:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233340.699292:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233340.699297:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233340.699312:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607101632/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233340.699322:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233340/real 1761233340] req@000000002ef1f54f x1846780607101440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233365 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233340.699349:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607101440/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233365 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233340.699357:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233340.699360:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233340.699364:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607101440:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233340.699391:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101504:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233340.699423:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233340/real 1761233340] req@0000000099514377 x1846780607101504/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233365 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233340.699436:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607101504/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233365 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233340.699443:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233340.699446:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233340.699449:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101504:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233340.699455:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101568:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233340.699493:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233340/real 1761233340] req@000000005a096598 x1846780607101568/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233365 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233340.699504:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607101568/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233365 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233340.699514:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233340.699517:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233340.699520:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101568:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233340.699526:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101632:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233340.699551:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233340/real 1761233340] req@00000000271a9d8e x1846780607101632/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233365 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233340.699562:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607101632/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233365 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233340.699570:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233340.699573:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233340.699576:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101632:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233341.147063:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233342.171140:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233343.195007:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233344.219070:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233345.243014:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233345.819059:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233345.819073:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233345.819077:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233345.819092:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6218 00000100:00080000:1.0:1761233345.819097:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233345.819107:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233345.819162:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607101696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233345.819216:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233345.819220:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233345.819221:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233345.819227:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6218 00000100:00080000:1.0:1761233345.819230:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233345.819233:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233345.819244:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233345.819260:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607101760/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233345.819268:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233345.819272:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233345.819273:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233345.819276:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6218 00000100:00080000:1.0:1761233345.819278:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233345.819281:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233345.819296:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233345.819303:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607101824/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233345.819310:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233345.819314:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233345.819316:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233345.819320:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6218 00000100:00080000:1.0:1761233345.819321:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233345.819324:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233345.819328:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233345.819336:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607101888/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233345.819364:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607101696:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233345.819480:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233345/real 1761233345] req@00000000659d78e3 x1846780607101696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233370 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233345.819499:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607101696/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233370 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233345.819510:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233345.819513:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233345.819517:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607101696:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233345.819524:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101760:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233345.819547:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233345/real 1761233345] req@00000000db10af3c x1846780607101760/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233370 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233345.819557:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607101760/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233370 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233345.819563:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233345.819565:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233345.819568:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101760:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233345.819572:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101824:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233345.819590:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233345/real 1761233345] req@00000000d07717c4 x1846780607101824/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233370 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233345.819599:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607101824/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233370 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233345.819606:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233345.819608:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233345.819610:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101824:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233345.819639:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101888:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233345.819656:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233345/real 1761233345] req@00000000b72feabb x1846780607101888/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233370 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233345.819665:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607101888/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233370 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233345.819672:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233345.819675:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233345.819678:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607101888:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233346.267140:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233347.291052:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233348.315058:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233349.339124:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233350.363059:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233350.939132:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233350.939149:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233350.939155:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233350.939173:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6223 00000100:00080000:2.0:1761233350.939178:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233350.939189:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233350.939249:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607101952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233350.939298:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233350.939303:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233350.939306:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233350.939312:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6223 00000100:00080000:2.0:1761233350.939316:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233350.939321:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233350.939335:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233350.939351:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607102016/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233350.939366:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761233350.939368:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607101952:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:2.0:1761233350.939371:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233350.939374:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233350.939387:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6223 00000100:00080000:2.0:1761233350.939393:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233350.939397:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233350.939406:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233350.939424:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607102080/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233350.939434:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233350.939439:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233350.939440:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233350.939445:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6223 00000100:00080000:2.0:1761233350.939449:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233350.939453:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233350.939459:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233350.939469:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607102144/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233350.939492:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233350/real 1761233350] req@0000000094135f44 x1846780607101952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233375 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233350.939512:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607101952/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233375 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233350.939522:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233350.939526:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233350.939530:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607101952:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233350.939554:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102016:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233350.939576:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233350/real 1761233350] req@0000000091106a97 x1846780607102016/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233375 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233350.939584:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607102016/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233375 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233350.939589:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233350.939591:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233350.939593:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102016:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233350.939597:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102080:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233350.939611:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233350/real 1761233350] req@0000000064978143 x1846780607102080/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233375 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233350.939618:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607102080/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233375 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233350.939626:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233350.939627:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233350.939629:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102080:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233350.939633:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102144:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233350.939646:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233350/real 1761233350] req@00000000c069b612 x1846780607102144/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233375 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233350.939652:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607102144/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233375 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233350.939658:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233350.939660:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233350.939662:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102144:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233351.387184:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233352.411171:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233353.435024:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233354.459045:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233355.483168:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233356.058934:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233356.058946:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233356.058951:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233356.058963:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6228 00000100:00080000:1.0:1761233356.058967:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233356.058974:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233356.059017:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607102208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233356.059044:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233356.059048:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233356.059049:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233356.059054:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6228 00000100:00080000:1.0:1761233356.059056:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233356.059059:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233356.059069:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233356.059080:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607102272/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233356.059084:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607102208:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761233356.059105:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233356.059114:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233356.059116:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233356.059122:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6228 00000100:00080000:1.0:1761233356.059125:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233356.059130:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233356.059137:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233356.059153:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607102336/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233356.059165:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233356.059169:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233356.059172:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233356.059176:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6228 00000100:00080000:1.0:1761233356.059178:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233356.059182:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233356.059187:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233356.059189:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233356/real 1761233356] req@00000000b72feabb x1846780607102208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233381 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233356.059201:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607102400/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233356.059268:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607102208/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233381 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233356.059289:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233356.059295:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233356.059302:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607102208:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233356.059346:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102272:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233356.059384:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233356/real 1761233356] req@00000000d07717c4 x1846780607102272/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233381 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233356.059397:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607102272/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233381 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233356.059422:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233356.059424:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233356.059427:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102272:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233356.059432:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102336:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233356.059449:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233356/real 1761233356] req@00000000db10af3c x1846780607102336/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233381 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233356.059457:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607102336/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233381 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233356.059464:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233356.059465:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233356.059467:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102336:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233356.059470:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102400:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233356.059493:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233356/real 1761233356] req@00000000659d78e3 x1846780607102400/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233381 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233356.059501:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607102400/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233381 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233356.059539:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233356.059541:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233356.059544:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102400:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233356.507068:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233357.531158:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233358.555045:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233359.578982:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233360.602995:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233361.179001:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233361.179011:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233361.179015:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233361.179027:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6233 00000100:00080000:2.0:1761233361.179030:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233361.179038:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233361.179087:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607102464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233361.179118:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233361.179122:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233361.179123:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233361.179127:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6233 00000100:00080000:2.0:1761233361.179129:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233361.179132:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233361.179141:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233361.179153:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607102528/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233361.179162:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233361.179166:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1761233361.179168:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607102464:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761233361.179170:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233361.179181:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6233 00000100:00080000:2.0:1761233361.179184:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233361.179187:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233361.179191:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233361.179205:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607102592/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233361.179213:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233361.179217:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233361.179218:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233361.179222:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6233 00000100:00080000:2.0:1761233361.179223:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233361.179226:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233361.179229:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233361.179239:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607102656/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233361.179273:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233361/real 1761233361] req@0000000017386078 x1846780607102464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233386 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233361.179291:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607102464/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233386 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233361.179301:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233361.179303:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233361.179308:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607102464:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233361.179331:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102528:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233361.179354:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233361/real 1761233361] req@00000000f38f27df x1846780607102528/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233386 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233361.179364:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607102528/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233386 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233361.179370:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233361.179372:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233361.179374:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102528:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233361.179379:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102592:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233361.179396:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233361/real 1761233361] req@00000000c069b612 x1846780607102592/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233386 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233361.179405:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607102592/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233386 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233361.179413:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233361.179415:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233361.179418:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102592:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233361.179422:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102656:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233361.179436:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233361/real 1761233361] req@0000000064978143 x1846780607102656/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233386 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233361.179444:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607102656/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233386 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233361.179450:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233361.179452:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233361.179454:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102656:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233361.627101:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233362.650996:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233363.674980:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233364.699051:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233365.723048:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233366.298937:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233366.298959:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233366.298964:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233366.298977:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6238 00000100:00080000:2.0:1761233366.298982:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233366.298990:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233366.299036:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607102720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233366.299081:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233366.299086:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233366.299088:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233366.299093:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6238 00000100:00080000:2.0:1761233366.299095:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233366.299098:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233366.299108:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233366.299123:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607102784/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233366.299133:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233366.299136:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233366.299138:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233366.299142:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6238 00000100:00080000:2.0:1761233366.299144:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233366.299148:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233366.299152:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233366.299163:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607102848/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233366.299174:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233366.299178:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233366.299180:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233366.299186:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6238 00000100:00080000:2.0:1761233366.299189:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233366.299194:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233366.299200:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233366.299209:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607102912/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233366.299246:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607102720:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233366.299411:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233366/real 1761233366] req@0000000091106a97 x1846780607102720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233391 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233366.299435:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607102720/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233391 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233366.299450:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233366.299455:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233366.299460:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607102720:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233366.299469:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102784:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233366.299502:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233366/real 1761233366] req@0000000094135f44 x1846780607102784/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233391 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233366.299515:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607102784/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233391 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233366.299523:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233366.299526:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233366.299529:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102784:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233366.299536:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102848:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233366.299559:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233366/real 1761233366] req@0000000064978143 x1846780607102848/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233391 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233366.299571:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607102848/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233391 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233366.299581:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233366.299583:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233366.299586:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102848:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233366.299625:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102912:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233366.299649:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233366/real 1761233366] req@00000000c069b612 x1846780607102912/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233391 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233366.299660:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607102912/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233391 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233366.299668:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233366.299672:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233366.299675:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607102912:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233366.747018:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233367.771084:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233368.795050:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233369.819041:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233370.843021:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233371.418934:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233371.418947:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233371.418951:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233371.418963:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6243 00000100:00080000:2.0:1761233371.418968:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233371.418976:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233371.419016:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607102976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233371.419048:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233371.419051:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233371.419052:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233371.419055:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6243 00000100:00080000:2.0:1761233371.419057:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233371.419060:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233371.419068:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233371.419079:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607103040/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233371.419086:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233371.419089:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233371.419091:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233371.419094:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6243 00000100:00080000:2.0:1761233371.419096:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233371.419127:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233371.419134:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233371.419138:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607102976:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233371.419162:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607103104/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233371.419177:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233371.419181:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233371.419184:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233371.419187:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6243 00000100:00080000:2.0:1761233371.419189:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233371.419194:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233371.419217:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233371.419230:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607103168/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233371.419245:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233371/real 1761233371] req@00000000f38f27df x1846780607102976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233396 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233371.419260:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607102976/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233396 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233371.419268:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233371.419271:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233371.419274:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607102976:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233371.419280:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103040:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233371.419314:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233371/real 1761233371] req@0000000017386078 x1846780607103040/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233396 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233371.419321:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607103040/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233396 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233371.419325:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233371.419326:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233371.419328:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103040:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233371.419345:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103104:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233371.419358:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233371/real 1761233371] req@00000000c069b612 x1846780607103104/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233396 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233371.419370:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607103104/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233396 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233371.419375:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233371.419376:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233371.419377:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103104:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233371.419381:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103168:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233371.419391:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233371/real 1761233371] req@0000000064978143 x1846780607103168/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233396 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233371.419397:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607103168/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233396 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233371.419402:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233371.419403:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233371.419405:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103168:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233371.867122:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233372.891085:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233373.915122:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233374.939034:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233375.963081:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233376.539001:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233376.539010:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233376.539014:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233376.539023:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6248 00000100:00080000:1.0:1761233376.539026:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233376.539032:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233376.539068:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607103232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233376.539092:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233376.539095:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233376.539097:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233376.539100:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6248 00000100:00080000:1.0:1761233376.539102:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233376.539104:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233376.539110:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233376.539119:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607103296/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233376.539127:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233376.539129:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233376.539131:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761233376.539133:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607103232:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761233376.539136:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6248 00000100:00080000:1.0:1761233376.539141:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233376.539144:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233376.539147:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233376.539155:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607103360/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233376.539161:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233376.539164:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233376.539165:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233376.539168:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6248 00000100:00080000:1.0:1761233376.539169:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233376.539171:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233376.539174:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233376.539183:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607103424/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233376.539222:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233376/real 1761233376] req@00000000659d78e3 x1846780607103232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233401 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233376.539237:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607103232/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233401 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233376.539244:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233376.539247:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233376.539250:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607103232:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233376.539270:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103296:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233376.539287:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233376/real 1761233376] req@00000000db10af3c x1846780607103296/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233401 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233376.539294:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607103296/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233401 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233376.539299:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233376.539300:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233376.539302:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103296:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233376.539306:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103360:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233376.539319:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233376/real 1761233376] req@00000000d07717c4 x1846780607103360/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233401 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233376.539325:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607103360/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233401 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233376.539330:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233376.539332:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233376.539333:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103360:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233376.539336:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103424:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233376.539348:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233376/real 1761233376] req@00000000b72feabb x1846780607103424/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233401 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233376.539354:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607103424/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233401 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233376.539360:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233376.539362:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233376.539363:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103424:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233376.987082:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233378.010987:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233379.034984:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233380.059023:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233381.083120:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233381.659002:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233381.659014:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233381.659018:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233381.659030:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6254 00000100:00080000:2.0:1761233381.659034:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233381.659042:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233381.659076:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607103488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233381.659102:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233381.659105:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233381.659106:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233381.659109:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6254 00000100:00080000:2.0:1761233381.659111:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233381.659114:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233381.659121:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233381.659131:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607103552/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233381.659140:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761233381.659143:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607103488:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761233381.659144:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233381.659148:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233381.659155:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6254 00000100:00080000:2.0:1761233381.659156:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233381.659159:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233381.659162:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233381.659176:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607103616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233381.659182:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233381.659185:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233381.659186:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233381.659189:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6254 00000100:00080000:2.0:1761233381.659190:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233381.659193:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233381.659196:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233381.659203:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607103680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233381.659237:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233381/real 1761233381] req@0000000094135f44 x1846780607103488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233406 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233381.659251:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607103488/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233406 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233381.659259:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233381.659262:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233381.659265:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607103488:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233381.659285:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103552:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233381.659304:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233381/real 1761233381] req@0000000091106a97 x1846780607103552/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233406 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233381.659312:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607103552/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233406 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233381.659317:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233381.659319:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233381.659321:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103552:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233381.659324:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103616:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233381.659338:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233381/real 1761233381] req@0000000064978143 x1846780607103616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233406 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233381.659350:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607103616/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233406 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233381.659356:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233381.659357:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233381.659359:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103616:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233381.659362:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103680:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233381.659374:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233381/real 1761233381] req@00000000c069b612 x1846780607103680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233406 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233381.659381:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607103680/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233406 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233381.659386:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233381.659387:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233381.659389:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103680:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233382.106999:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233383.130985:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233384.154967:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233385.179078:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233386.202979:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233386.778986:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233386.778995:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233386.778998:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233386.779008:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6259 00000100:00080000:1.0:1761233386.779010:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233386.779015:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233386.779047:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607103744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233386.779070:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233386.779074:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233386.779076:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233386.779080:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6259 00000100:00080000:1.0:1761233386.779082:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233386.779094:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233386.779103:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233386.779107:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607103744:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233386.779115:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607103808/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233386.779125:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233386.779133:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233386.779135:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233386.779138:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6259 00000100:00080000:1.0:1761233386.779140:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233386.779143:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233386.779147:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233386.779156:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607103872/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233386.779164:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233386.779166:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233386.779168:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233386.779171:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6259 00000100:00080000:1.0:1761233386.779173:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233386.779192:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233386.779197:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233386.779206:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607103936/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233386.779214:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103808:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233386.779244:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103872:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233386.779277:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103936:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233387.227041:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233388.251027:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233389.275056:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761233389.851179:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233386/real 1761233389] req@00000000659d78e3 x1846780607103936/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233411 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233389.851209:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607103936/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233411 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233389.851220:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233389.851225:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233389.851229:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103936:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233389.851235:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233386/real 1761233389] req@00000000d07717c4 x1846780607103808/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233411 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233389.851246:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607103808/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233411 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233389.851253:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233389.851255:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233389.851257:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103808:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233389.851262:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233386/real 1761233389] req@00000000db10af3c x1846780607103872/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233411 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233389.851273:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607103872/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233411 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233389.851280:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233389.851282:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233389.851285:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607103872:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233389.851290:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233386/real 1761233389] req@00000000b72feabb x1846780607103744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233411 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233389.851303:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607103744/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233411 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233389.851310:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233389.851312:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233389.851315:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607103744:192.168.201.104@tcp:250:kworker/u8:1.0 00010000:00010000:3.0:1761233390.299097:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233391.323167:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233391.898976:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233391.898990:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233391.898993:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233391.899001:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6264 00000100:00080000:3.0:1761233391.899004:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233391.899009:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233391.899035:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607104000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233391.899057:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233391.899060:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233391.899061:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233391.899064:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6264 00000100:00080000:3.0:1761233391.899065:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233391.899067:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233391.899075:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233391.899088:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607104064/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233391.899110:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233391.899114:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233391.899116:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761233391.899122:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607104000:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:3.0:1761233391.899122:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6264 00000100:00080000:3.0:1761233391.899126:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233391.899132:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233391.899139:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233391.899153:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607104128/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233391.899162:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233391.899166:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233391.899167:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233391.899171:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6264 00000100:00080000:3.0:1761233391.899173:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233391.899176:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233391.899181:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233391.899192:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607104192/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233391.899216:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233391/real 1761233391] req@000000007361665d x1846780607104000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233416 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233391.899233:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607104000/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233416 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233391.899242:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233391.899246:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233391.899249:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607104000:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233391.899256:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104064:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233391.899281:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233391/real 1761233391] req@00000000f87d0618 x1846780607104064/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233416 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233391.899291:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607104064/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233416 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233391.899313:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233391.899316:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233391.899319:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104064:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233391.899347:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104128:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233391.899373:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233391/real 1761233391] req@00000000c7e4e432 x1846780607104128/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233416 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233391.899383:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607104128/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233416 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233391.899390:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233391.899392:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233391.899395:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104128:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233391.899400:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104192:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233391.899421:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233391/real 1761233391] req@0000000031e043f4 x1846780607104192/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233416 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233391.899599:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607104192/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233416 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233391.899609:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233391.899611:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233391.899613:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104192:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233392.347007:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233393.371028:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233394.395035:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233395.419003:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233396.443080:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233397.018982:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233397.018992:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233397.018995:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233397.019005:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6269 00000100:00080000:0.0:1761233397.019008:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233397.019014:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233397.019046:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607104256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233397.019069:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233397.019072:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233397.019073:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233397.019076:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6269 00000100:00080000:0.0:1761233397.019079:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233397.019081:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233397.019096:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233397.019108:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607104320/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233397.019114:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233397.019117:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233397.019117:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233397.019120:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6269 00000100:00080000:0.0:1761233397.019121:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233397.019123:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233397.019125:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233397.019131:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607104384/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233397.019137:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233397.019139:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233397.019140:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233397.019142:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6269 00000100:00080000:0.0:1761233397.019144:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233397.019146:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233397.019148:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233397.019157:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607104448/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233397.019194:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607104256:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233397.019322:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233397/real 1761233397] req@0000000023ab3a3f x1846780607104256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233422 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233397.019338:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607104256/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233422 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233397.019348:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233397.019351:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233397.019354:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607104256:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233397.019359:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104320:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233397.019376:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233397/real 1761233397] req@0000000001b2f813 x1846780607104320/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233422 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233397.019383:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607104320/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233422 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233397.019387:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233397.019388:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233397.019390:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104320:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233397.019393:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104384:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233397.019406:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233397/real 1761233397] req@00000000275aeee5 x1846780607104384/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233422 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233397.019412:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607104384/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233422 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233397.019417:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233397.019419:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233397.019420:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104384:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233397.019423:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104448:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233397.019435:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233397/real 1761233397] req@000000000243251f x1846780607104448/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233422 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233397.019441:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607104448/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233422 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233397.019446:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233397.019447:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233397.019449:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104448:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233397.467026:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233398.491133:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233399.515068:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233400.539179:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233401.563053:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233402.139037:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233402.139050:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233402.139054:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233402.139066:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6274 00000100:00080000:3.0:1761233402.139070:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233402.139078:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233402.139125:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607104512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233402.139166:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233402.139169:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233402.139170:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233402.139174:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6274 00000100:00080000:3.0:1761233402.139176:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233402.139178:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233402.139187:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233402.139196:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607104576/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233402.139202:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233402.139205:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233402.139207:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233402.139210:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6274 00000100:00080000:3.0:1761233402.139211:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233402.139214:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233402.139219:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233402.139229:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607104640/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233402.139236:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233402.139239:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233402.139240:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233402.139243:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6274 00000100:00080000:3.0:1761233402.139244:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233402.139246:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233402.139249:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233402.139256:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607104512:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761233402.139260:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607104704/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233402.139376:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233402/real 1761233402] req@000000005a9bdefd x1846780607104512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233427 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233402.139396:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607104512/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233427 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233402.139406:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233402.139410:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233402.139414:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607104512:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233402.139421:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104576:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233402.139444:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233402/real 1761233402] req@0000000086ccb527 x1846780607104576/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233427 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233402.139454:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607104576/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233427 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233402.139467:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233402.139469:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233402.139472:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104576:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233402.139498:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104640:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233402.139518:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233402/real 1761233402] req@00000000bd684ccc x1846780607104640/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233427 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233402.139527:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607104640/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233427 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233402.139535:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233402.139537:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233402.139539:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104640:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233402.139544:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104704:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233402.139561:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233402/real 1761233402] req@0000000018a8f5da x1846780607104704/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233427 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233402.139569:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607104704/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233427 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233402.139576:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233402.139578:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233402.139580:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104704:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233402.587113:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233403.611184:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233404.635075:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233405.658968:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233406.683113:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233407.259005:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233407.259015:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233407.259018:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233407.259027:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6279 00000100:00080000:3.0:1761233407.259031:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233407.259038:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233407.259090:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607104768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233407.259117:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233407.259122:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233407.259124:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233407.259130:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6279 00000100:00080000:3.0:1761233407.259133:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233407.259137:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233407.259146:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233407.259161:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607104832/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233407.259173:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233407.259176:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233407.259179:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761233407.259181:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607104768:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:3.0:1761233407.259187:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6279 00000100:00080000:3.0:1761233407.259192:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233407.259197:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233407.259203:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233407.259240:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607104896/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233407.259253:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233407.259257:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233407.259259:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233407.259264:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6279 00000100:00080000:3.0:1761233407.259267:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233407.259271:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233407.259277:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233407.259296:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607104960/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233407.259320:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233407/real 1761233407] req@00000000daa39bb1 x1846780607104768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233432 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233407.259346:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607104768/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233432 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233407.259357:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233407.259360:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233407.259364:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607104768:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233407.259387:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104832:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233407.259414:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233407/real 1761233407] req@000000003ad149cc x1846780607104832/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233432 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233407.259426:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607104832/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233432 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233407.259433:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233407.259436:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233407.259439:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104832:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233407.259445:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104896:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233407.259469:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233407/real 1761233407] req@0000000088473b3e x1846780607104896/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233432 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233407.259481:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607104896/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233432 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233407.259495:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233407.259497:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233407.259499:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104896:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233407.259505:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104960:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233407.259539:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233407/real 1761233407] req@000000004e3f612a x1846780607104960/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233432 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233407.259552:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607104960/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233432 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233407.259561:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233407.259563:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233407.259566:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607104960:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233407.707019:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233408.731099:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233409.754981:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233410.779057:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233411.803008:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233412.379009:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233412.379017:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233412.379020:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233412.379029:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6284 00000100:00080000:1.0:1761233412.379032:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233412.379038:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233412.379071:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607105024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233412.379093:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233412.379096:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233412.379097:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233412.379100:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6284 00000100:00080000:1.0:1761233412.379101:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233412.379103:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233412.379109:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233412.379116:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607105088/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233412.379121:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233412.379123:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233412.379124:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233412.379127:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6284 00000100:00080000:1.0:1761233412.379128:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233412.379131:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233412.379135:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233412.379140:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607105152/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233412.379146:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233412.379148:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233412.379149:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761233412.379153:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607105024:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761233412.379155:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6284 00000100:00080000:1.0:1761233412.379159:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233412.379162:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233412.379165:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233412.379172:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607105216/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233412.379230:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233412/real 1761233412] req@00000000b72feabb x1846780607105024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233437 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233412.379244:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607105024/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233437 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233412.379251:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233412.379254:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233412.379256:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607105024:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233412.379261:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105088:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233412.379275:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233412/real 1761233412] req@00000000db10af3c x1846780607105088/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233437 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233412.379281:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607105088/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233437 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233412.379286:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233412.379287:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233412.379288:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105088:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233412.379305:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105152:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233412.379317:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233412/real 1761233412] req@00000000d07717c4 x1846780607105152/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233437 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233412.379323:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607105152/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233437 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233412.379328:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233412.379329:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233412.379330:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105152:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233412.379333:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105216:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233412.379343:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233412/real 1761233412] req@00000000659d78e3 x1846780607105216/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233437 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233412.379349:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607105216/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233437 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233412.379354:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233412.379355:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233412.379357:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105216:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233412.827028:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233413.851028:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233414.875043:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233415.899021:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233416.923030:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233417.498946:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233417.498957:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233417.498961:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233417.498972:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6289 00000100:00080000:0.0:1761233417.498977:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233417.498984:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233417.499021:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607105280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233417.499047:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233417.499051:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233417.499053:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233417.499057:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6289 00000100:00080000:0.0:1761233417.499059:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233417.499063:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233417.499073:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233417.499085:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607105344/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233417.499091:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607105280:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761233417.499098:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233417.499103:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233417.499104:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233417.499109:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6289 00000100:00080000:0.0:1761233417.499112:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233417.499115:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233417.499120:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233417.499133:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607105408/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233417.499142:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233417.499146:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233417.499147:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233417.499150:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6289 00000100:00080000:0.0:1761233417.499153:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233417.499156:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233417.499160:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233417.499172:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607105472/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233417.499184:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233417/real 1761233417] req@00000000b8f2be78 x1846780607105280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233442 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233417.499205:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607105280/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233442 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233417.499216:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233417.499219:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233417.499223:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607105280:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233417.499245:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105344:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233417.499275:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233417/real 1761233417] req@00000000fcce404c x1846780607105344/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233442 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233417.499287:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607105344/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233442 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233417.499295:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233417.499298:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233417.499301:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105344:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233417.499308:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105408:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233417.499332:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233417/real 1761233417] req@000000007f60e210 x1846780607105408/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233442 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233417.499344:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607105408/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233442 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233417.499353:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233417.499355:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233417.499358:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105408:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233417.499364:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105472:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233417.499390:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233417/real 1761233417] req@000000000243251f x1846780607105472/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233442 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233417.499401:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607105472/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233442 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233417.499409:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233417.499411:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233417.499414:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105472:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761233417.947066:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233418.971008:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233419.995010:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233421.019045:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233422.042984:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233422.618981:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233422.618991:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233422.618994:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233422.619004:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6295 00000100:00080000:2.0:1761233422.619007:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233422.619013:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233422.619044:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607105536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233422.619070:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233422.619073:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233422.619074:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233422.619077:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6295 00000100:00080000:2.0:1761233422.619079:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233422.619081:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233422.619088:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233422.619097:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607105600/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233422.619104:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233422.619107:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233422.619108:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233422.619111:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6295 00000100:00080000:2.0:1761233422.619112:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233422.619115:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233422.619118:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233422.619126:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607105536:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233422.619129:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607105664/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233422.619140:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233422.619143:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233422.619144:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233422.619147:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6295 00000100:00080000:2.0:1761233422.619148:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233422.619151:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233422.619154:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233422.619163:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607105728/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233422.619237:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233422/real 1761233422] req@0000000017386078 x1846780607105536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233447 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233422.619257:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607105536/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233447 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233422.619267:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233422.619270:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233422.619275:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607105536:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233422.619282:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105600:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233422.619315:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233422/real 1761233422] req@00000000f38f27df x1846780607105600/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233447 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233422.619337:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607105600/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233447 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233422.619346:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233422.619349:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233422.619352:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105600:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233422.619373:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105664:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233422.619395:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233422/real 1761233422] req@00000000c069b612 x1846780607105664/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233447 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233422.619403:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607105664/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233447 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233422.619409:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233422.619411:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233422.619413:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105664:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233422.619416:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105728:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233422.619429:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233422/real 1761233422] req@0000000064978143 x1846780607105728/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233447 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233422.619436:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607105728/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233447 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233422.619442:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233422.619443:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233422.619446:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105728:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761233423.066998:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233424.091020:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233425.114973:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233426.138972:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233427.163053:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233427.738921:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233427.738931:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233427.738933:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233427.738942:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6300 00000100:00080000:2.0:1761233427.738945:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233427.738951:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233427.738993:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607105792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233427.739015:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233427.739017:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233427.739018:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233427.739021:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6300 00000100:00080000:2.0:1761233427.739022:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233427.739024:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233427.739031:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233427.739040:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607105856/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233427.739045:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233427.739047:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233427.739049:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233427.739052:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6300 00000100:00080000:2.0:1761233427.739053:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233427.739055:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233427.739057:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233427.739063:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607105920/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233427.739067:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233427.739069:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233427.739071:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233427.739074:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6300 00000100:00080000:2.0:1761233427.739075:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233427.739077:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233427.739080:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233427.739086:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607105984/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233427.739168:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607105792:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233427.739300:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233427/real 1761233427] req@0000000091106a97 x1846780607105792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233452 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233427.739317:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607105792/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233452 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233427.739327:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233427.739331:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233427.739335:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607105792:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233427.739342:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105856:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233427.739364:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233427/real 1761233427] req@0000000094135f44 x1846780607105856/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233452 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233427.739373:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607105856/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233452 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233427.739379:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233427.739381:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233427.739384:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105856:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233427.739388:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105920:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233427.739405:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233427/real 1761233427] req@0000000064978143 x1846780607105920/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233452 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233427.739413:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607105920/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233452 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233427.739421:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233427.739424:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233427.739426:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105920:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233427.739430:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105984:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233427.739445:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233427/real 1761233427] req@00000000c069b612 x1846780607105984/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233452 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233427.739452:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607105984/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233452 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233427.739459:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233427.739461:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233427.739463:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607105984:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761233428.186986:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233429.211016:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233430.235076:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233431.259038:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233432.283004:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233432.858992:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233432.859002:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233432.859006:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233432.859016:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6305 00000100:00080000:3.0:1761233432.859019:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233432.859025:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233432.859063:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607106048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233432.859088:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233432.859091:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233432.859093:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233432.859096:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6305 00000100:00080000:3.0:1761233432.859098:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233432.859100:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233432.859107:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233432.859126:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607106112/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233432.859133:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233432.859136:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233432.859138:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233432.859141:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6305 00000100:00080000:3.0:1761233432.859143:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233432.859146:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233432.859151:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233432.859163:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607106048:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761233432.859165:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607106176/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233432.859190:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233432.859194:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233432.859196:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233432.859200:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6305 00000100:00080000:3.0:1761233432.859203:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233432.859207:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233432.859213:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233432.859227:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607106240/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233432.859260:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233432/real 1761233432] req@000000004e3f612a x1846780607106048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233457 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233432.859276:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607106048/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233457 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233432.859284:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233432.859286:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233432.859289:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607106048:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233432.859294:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106112:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233432.859315:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233432/real 1761233432] req@0000000088473b3e x1846780607106112/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233457 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233432.859323:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607106112/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233457 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233432.859329:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233432.859330:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233432.859332:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106112:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233432.859354:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106176:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233432.859370:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233432/real 1761233432] req@000000003ad149cc x1846780607106176/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233457 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233432.859377:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607106176/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233457 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233432.859384:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233432.859386:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233432.859388:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106176:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233432.859391:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106240:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233432.859411:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233432/real 1761233432] req@00000000daa39bb1 x1846780607106240/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233457 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233432.859417:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607106240/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233457 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233432.859423:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233432.859425:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233432.859427:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106240:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761233433.307038:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233434.331037:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233435.355028:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233436.379144:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233437.403065:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233437.978973:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233437.978982:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233437.978985:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233437.978994:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6310 00000100:00080000:2.0:1761233437.978997:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233437.979003:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233437.979036:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607106304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233437.979059:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233437.979061:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233437.979062:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233437.979065:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6310 00000100:00080000:2.0:1761233437.979067:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233437.979069:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233437.979076:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233437.979091:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607106368/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233437.979097:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233437.979099:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233437.979100:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233437.979101:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6310 00000100:00080000:2.0:1761233437.979103:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233437.979104:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233437.979107:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233437.979113:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607106432/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233437.979118:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233437.979120:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233437.979121:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233437.979122:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6310 00000100:00080000:2.0:1761233437.979124:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233437.979130:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233437.979134:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233437.979140:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607106496/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233437.979206:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607106304:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233437.979334:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233437/real 1761233437] req@00000000f38f27df x1846780607106304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233462 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233437.979352:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607106304/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233462 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233437.979377:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233437.979381:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233437.979386:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607106304:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233437.979393:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106368:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233437.979423:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233437/real 1761233437] req@0000000017386078 x1846780607106368/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233462 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233437.979433:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607106368/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233462 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233437.979439:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233437.979441:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233437.979444:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106368:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233437.979449:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106432:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233437.979467:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233437/real 1761233437] req@00000000c069b612 x1846780607106432/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233462 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233437.979482:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607106432/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233462 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233437.979488:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233437.979490:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233437.979492:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106432:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233437.979496:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106496:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233437.979512:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233437/real 1761233437] req@0000000064978143 x1846780607106496/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233462 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233437.979521:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607106496/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233462 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233437.979527:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233437.979529:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233437.979531:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106496:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761233438.427135:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233439.451141:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233440.475071:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233441.499182:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233442.523007:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233443.098950:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233443.098959:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233443.098962:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233443.098971:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6315 00000100:00080000:2.0:1761233443.098974:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233443.098979:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233443.099011:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607106560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233443.099033:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233443.099035:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233443.099036:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233443.099039:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6315 00000100:00080000:2.0:1761233443.099040:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233443.099042:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233443.099048:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233443.099057:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607106624/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233443.099062:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233443.099065:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233443.099065:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233443.099068:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6315 00000100:00080000:2.0:1761233443.099069:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233443.099071:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233443.099073:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233443.099079:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607106688/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233443.099083:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233443.099085:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233443.099086:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233443.099088:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6315 00000100:00080000:2.0:1761233443.099089:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233443.099091:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233443.099094:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233443.099100:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607106752/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233443.099115:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607106560:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233443.099237:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233443/real 1761233443] req@0000000094135f44 x1846780607106560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233468 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233443.099255:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607106560/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233468 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233443.099265:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233443.099268:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233443.099271:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607106560:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233443.099277:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106624:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233443.099298:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233443/real 1761233443] req@0000000091106a97 x1846780607106624/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233468 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233443.099305:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607106624/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233468 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233443.099310:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233443.099311:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233443.099313:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106624:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233443.099316:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106688:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233443.099327:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233443/real 1761233443] req@0000000064978143 x1846780607106688/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233468 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233443.099333:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607106688/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233468 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233443.099338:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233443.099339:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233443.099340:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106688:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233443.099359:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106752:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233443.099371:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233443/real 1761233443] req@00000000c069b612 x1846780607106752/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233468 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233443.099376:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607106752/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233468 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233443.099381:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233443.099382:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233443.099383:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106752:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761233443.547182:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233444.570997:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233445.595068:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233446.619125:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233447.643016:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233448.219015:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233448.219027:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233448.219030:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233448.219041:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6320 00000100:00080000:0.0:1761233448.219045:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233448.219052:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233448.219094:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607106816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233448.219134:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233448.219139:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233448.219142:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233448.219147:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6320 00000100:00080000:0.0:1761233448.219150:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233448.219155:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233448.219169:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233448.219188:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607106880/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233448.219202:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233448.219208:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233448.219210:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233448.219219:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6320 00000100:00100000:1.0:1761233448.219219:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607106816:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761233448.219223:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233448.219232:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233448.219240:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233448.219260:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607106944/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233448.219272:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233448.219275:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233448.219277:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233448.219281:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6320 00000100:00080000:0.0:1761233448.219283:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233448.219285:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233448.219289:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233448.219299:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607107008/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233448.219359:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233448/real 1761233448] req@00000000275aeee5 x1846780607106816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233473 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233448.219383:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607106816/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233473 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233448.219396:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233448.219399:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233448.219404:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607106816:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233448.219457:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106880:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233448.219492:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233448/real 1761233448] req@0000000001b2f813 x1846780607106880/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233473 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233448.219502:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607106880/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233473 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233448.219510:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233448.219512:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233448.219514:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106880:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233448.219539:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106944:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233448.219559:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233448/real 1761233448] req@0000000023ab3a3f x1846780607106944/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233473 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233448.219569:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607106944/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233473 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233448.219577:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233448.219579:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233448.219581:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607106944:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233448.219586:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107008:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233448.219601:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233448/real 1761233448] req@00000000271a9d8e x1846780607107008/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233473 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233448.219615:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607107008/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233473 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233448.219622:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233448.219624:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233448.219626:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107008:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761233448.667090:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233449.691163:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233450.715009:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233451.739082:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233452.763022:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233453.339058:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233453.339080:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233453.339085:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233453.339101:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6325 00000100:00080000:3.0:1761233453.339106:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233453.339116:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233453.339171:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607107072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233453.339221:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233453.339225:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233453.339227:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233453.339231:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6325 00000100:00080000:3.0:1761233453.339233:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233453.339235:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233453.339246:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233453.339258:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607107136/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233453.339265:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233453.339267:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233453.339269:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233453.339272:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6325 00000100:00080000:3.0:1761233453.339273:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233453.339275:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233453.339279:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233453.339290:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607107200/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233453.339296:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233453.339299:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233453.339300:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233453.339303:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6325 00000100:00080000:3.0:1761233453.339305:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233453.339308:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233453.339312:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233453.339319:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607107264/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233453.339343:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607107072:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233453.339501:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107136:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233453.339549:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107200:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233453.339610:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107264:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761233453.787124:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233454.811075:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233455.835079:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761233456.411365:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233453/real 1761233456] req@000000005a9bdefd x1846780607107264/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233478 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233456.411417:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607107264/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233478 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233456.411432:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233456.411436:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233456.411441:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107264:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233456.411452:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233453/real 1761233456] req@0000000018a8f5da x1846780607107072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233478 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233456.411471:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607107072/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233478 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233456.411481:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233456.411485:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233456.411489:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607107072:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233456.411497:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233453/real 1761233456] req@00000000bd684ccc x1846780607107136/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233478 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233456.411513:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607107136/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233478 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233456.411523:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233456.411527:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233456.411531:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107136:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233456.411539:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233453/real 1761233456] req@0000000086ccb527 x1846780607107200/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233478 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233456.411565:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607107200/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233478 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233456.411576:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233456.411580:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233456.411584:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107200:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233456.859208:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233457.883135:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233458.459010:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233458.459020:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233458.459024:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233458.459034:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6330 00000100:00080000:2.0:1761233458.459037:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233458.459042:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233458.459073:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607107328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233458.459099:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233458.459102:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233458.459103:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233458.459107:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6330 00000100:00080000:2.0:1761233458.459108:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233458.459110:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233458.459117:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233458.459126:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607107392/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233458.459133:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233458.459135:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233458.459136:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233458.459138:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6330 00000100:00080000:2.0:1761233458.459140:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233458.459142:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233458.459145:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233458.459152:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607107456/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233458.459157:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233458.459161:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233458.459162:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233458.459164:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6330 00000100:00080000:2.0:1761233458.459166:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233458.459168:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233458.459171:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233458.459179:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607107520/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233458.459208:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607107328:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233458.459327:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233458/real 1761233458] req@0000000017386078 x1846780607107328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233483 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233458.459347:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607107328/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233483 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233458.459375:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233458.459379:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233458.459383:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607107328:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233458.459389:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107392:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233458.459418:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233458/real 1761233458] req@00000000f38f27df x1846780607107392/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233483 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233458.459429:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607107392/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233483 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233458.459436:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233458.459439:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233458.459442:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107392:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233458.459448:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107456:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233458.459483:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233458/real 1761233458] req@00000000c069b612 x1846780607107456/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233483 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233458.459494:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607107456/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233483 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233458.459503:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233458.459506:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233458.459510:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107456:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233458.459517:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107520:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233458.459548:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233458/real 1761233458] req@0000000064978143 x1846780607107520/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233483 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233458.459558:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607107520/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233483 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233458.459569:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233458.459572:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233458.459576:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107520:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761233458.907221:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233459.931002:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233460.955063:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233461.979012:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233463.003020:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233463.579023:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233463.579036:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233463.579041:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233463.579055:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6335 00000100:00080000:0.0:1761233463.579059:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233463.579069:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233463.579115:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607107584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233463.579152:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233463.579157:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233463.579159:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233463.579163:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6335 00000100:00080000:0.0:1761233463.579166:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233463.579217:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233463.579232:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233463.579257:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607107648/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233463.579276:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761233463.579280:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607107584:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761233463.579285:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233463.579290:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233463.579302:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6335 00000100:00080000:0.0:1761233463.579305:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233463.579310:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233463.579318:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233463.579338:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607107712/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233463.579350:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233463.579354:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233463.579357:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233463.579361:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6335 00000100:00080000:0.0:1761233463.579364:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233463.579367:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233463.579372:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233463.579387:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607107776/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233463.579433:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233463/real 1761233463] req@000000005a096598 x1846780607107584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233488 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233463.579467:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607107584/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233488 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233463.579488:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233463.579494:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233463.579500:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607107584:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233463.579533:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107648:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233463.579574:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233463/real 1761233463] req@0000000099514377 x1846780607107648/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233488 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233463.579588:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607107648/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233488 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233463.579596:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233463.579599:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233463.579603:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107648:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233463.579609:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107712:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233463.579632:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233463/real 1761233463] req@000000002ef1f54f x1846780607107712/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233488 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233463.579643:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607107712/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233488 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233463.579652:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233463.579654:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233463.579657:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107712:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233463.579663:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107776:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233463.579685:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233463/real 1761233463] req@00000000271a9d8e x1846780607107776/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233488 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233463.579696:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607107776/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233488 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233463.579705:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233463.579707:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233463.579710:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107776:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:2.0:1761233464.026960:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233465.051003:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233466.074984:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233467.099055:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233468.123016:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233468.699110:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233468.699124:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233468.699129:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233468.699141:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6341 00000100:00080000:3.0:1761233468.699145:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233468.699153:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233468.699197:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607107840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233468.699233:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233468.699237:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233468.699239:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233468.699243:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6341 00000100:00080000:3.0:1761233468.699245:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233468.699248:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233468.699257:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233468.699276:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607107904/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233468.699286:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233468.699289:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233468.699291:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233468.699294:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6341 00000100:00080000:3.0:1761233468.699296:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233468.699300:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233468.699305:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233468.699316:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607107968/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233468.699325:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233468.699330:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233468.699332:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233468.699336:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6341 00000100:00080000:3.0:1761233468.699338:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233468.699342:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233468.699346:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233468.699355:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607108032/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233468.699368:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607107840:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233468.699472:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233468/real 1761233468] req@0000000031e043f4 x1846780607107840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233493 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233468.699492:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607107840/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233493 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233468.699503:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233468.699507:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233468.699511:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607107840:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233468.699519:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107904:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233468.699543:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233468/real 1761233468] req@00000000c7e4e432 x1846780607107904/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233493 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233468.699555:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607107904/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233493 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233468.699562:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233468.699565:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233468.699568:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107904:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233468.699574:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107968:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233468.699594:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233468/real 1761233468] req@00000000f87d0618 x1846780607107968/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233493 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233468.699605:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607107968/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233493 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233468.699613:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233468.699616:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233468.699618:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607107968:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233468.699648:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108032:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233468.699669:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233468/real 1761233468] req@000000007361665d x1846780607108032/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233493 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233468.699680:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607108032/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233493 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233468.699688:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233468.699690:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233468.699692:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108032:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761233469.147098:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233470.171061:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233471.195128:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233472.219007:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233473.243046:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233473.819083:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233473.819094:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233473.819098:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233473.819108:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6346 00000100:00080000:0.0:1761233473.819111:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233473.819117:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233473.819152:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607108096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233473.819181:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233473.819185:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233473.819186:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233473.819190:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6346 00000100:00080000:0.0:1761233473.819192:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233473.819195:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233473.819203:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233473.819214:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607108160/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233473.819223:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233473.819226:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233473.819227:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233473.819231:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6346 00000100:00080000:0.0:1761233473.819232:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233473.819238:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:1.0:1761233473.819239:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607108096:192.168.201.104@tcp:250:kworker/u8:1.0 00000008:00100000:0.0:1761233473.819242:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233473.819253:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607108224/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233473.819261:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233473.819264:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233473.819265:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233473.819267:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6346 00000100:00080000:0.0:1761233473.819268:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233473.819270:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233473.819274:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233473.819282:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607108288/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233473.819327:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233473/real 1761233473] req@0000000023ab3a3f x1846780607108096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233498 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233473.819347:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607108096/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233498 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233473.819355:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233473.819357:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233473.819360:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607108096:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233473.819366:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108160:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233473.819391:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233473/real 1761233473] req@0000000001b2f813 x1846780607108160/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233498 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233473.819405:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607108160/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233498 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233473.819414:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233473.819417:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233473.819420:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108160:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233473.819450:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108224:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233473.819485:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233473/real 1761233473] req@00000000275aeee5 x1846780607108224/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233498 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233473.819499:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607108224/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233498 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233473.819509:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233473.819512:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233473.819516:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108224:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233473.819523:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108288:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233473.819555:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233473/real 1761233473] req@000000000243251f x1846780607108288/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233498 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233473.819568:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607108288/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233498 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233473.819578:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233473.819581:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233473.819584:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108288:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761233474.267069:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233475.291154:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233476.315006:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233477.338992:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233478.363080:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233478.938988:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233478.938997:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233478.939001:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233478.939010:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6351 00000100:00080000:3.0:1761233478.939013:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233478.939019:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233478.939052:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607108352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233478.939074:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233478.939077:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233478.939078:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233478.939080:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6351 00000100:00080000:3.0:1761233478.939082:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233478.939084:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233478.939090:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233478.939097:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607108416/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233478.939102:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233478.939105:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233478.939106:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233478.939109:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6351 00000100:00080000:3.0:1761233478.939110:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233478.939112:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233478.939115:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233478.939123:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607108352:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761233478.939126:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607108480/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233478.939134:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233478.939136:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233478.939136:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233478.939139:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6351 00000100:00080000:3.0:1761233478.939140:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233478.939143:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233478.939146:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233478.939153:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607108544/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233478.939203:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233478/real 1761233478] req@0000000090b40c66 x1846780607108352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233503 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233478.939216:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607108352/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233503 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233478.939223:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233478.939225:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233478.939228:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607108352:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233478.939233:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108416:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233478.939247:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233478/real 1761233478] req@0000000083f16a6b x1846780607108416/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233503 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233478.939253:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607108416/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233503 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233478.939262:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233478.939264:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233478.939265:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108416:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233478.939284:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108480:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233478.939298:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233478/real 1761233478] req@00000000a58ca11e x1846780607108480/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233503 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233478.939303:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607108480/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233503 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233478.939309:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233478.939310:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233478.939311:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108480:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233478.939315:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108544:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233478.939325:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233478/real 1761233478] req@000000001f8e688e x1846780607108544/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233503 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233478.939332:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607108544/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233503 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233478.939337:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233478.939338:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233478.939339:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108544:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761233479.386972:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233480.411104:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233481.435021:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233482.459038:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233483.483050:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233484.058995:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233484.059008:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233484.059012:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233484.059023:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6356 00000100:00080000:0.0:1761233484.059027:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233484.059036:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233484.059070:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607108608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233484.059095:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233484.059115:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233484.059116:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233484.059120:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6356 00000100:00080000:0.0:1761233484.059123:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233484.059125:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233484.059133:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233484.059145:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607108672/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233484.059153:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233484.059157:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233484.059158:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233484.059161:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6356 00000100:00080000:0.0:1761233484.059162:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233484.059165:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233484.059169:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233484.059170:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607108608:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:0.0:1761233484.059182:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607108736/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233484.059191:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233484.059194:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233484.059196:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233484.059200:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6356 00000100:00080000:0.0:1761233484.059202:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233484.059204:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233484.059208:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233484.059219:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607108800/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233484.059269:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233484/real 1761233484] req@000000007f60e210 x1846780607108608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233509 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233484.059286:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607108608/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233509 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233484.059295:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233484.059298:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233484.059302:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607108608:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233484.059308:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108672:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233484.059329:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233484/real 1761233484] req@00000000fcce404c x1846780607108672/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233509 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233484.059338:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607108672/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233509 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233484.059344:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233484.059346:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233484.059348:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108672:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:1.0:1761233484.059370:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108736:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233484.059389:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233484/real 1761233484] req@00000000b8f2be78 x1846780607108736/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233509 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233484.059398:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607108736/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233509 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233484.059405:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233484.059408:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233484.059410:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108736:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233484.059414:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108800:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761233484.059429:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233484/real 1761233484] req@000000000243251f x1846780607108800/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233509 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:1.0:1761233484.059438:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607108800/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233509 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:1.0:1761233484.059444:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233484.059446:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233484.059448:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108800:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761233484.507023:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233485.531125:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233486.554954:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233487.579015:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233488.603112:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233489.178949:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233489.178961:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233489.178965:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233489.178976:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6361 00000100:00080000:1.0:1761233489.178981:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233489.178986:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233489.179017:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607108864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233489.179038:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233489.179041:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233489.179042:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233489.179044:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6361 00000100:00080000:1.0:1761233489.179047:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233489.179049:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233489.179056:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233489.179066:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607108928/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233489.179076:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761233489.179080:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607108864:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761233489.179081:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233489.179083:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233489.179091:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6361 00000100:00080000:1.0:1761233489.179094:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233489.179097:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233489.179102:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233489.179114:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607108992/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233489.179124:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233489.179127:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233489.179128:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233489.179132:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6361 00000100:00080000:1.0:1761233489.179134:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233489.179136:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233489.179140:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233489.179148:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607109056/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233489.179174:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233489/real 1761233489] req@00000000659d78e3 x1846780607108864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233514 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233489.179194:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607108864/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233514 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233489.179204:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233489.179207:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233489.179212:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607108864:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233489.179233:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108928:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233489.179259:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233489/real 1761233489] req@00000000d07717c4 x1846780607108928/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233514 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233489.179270:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607108928/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233514 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233489.179278:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233489.179280:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233489.179284:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108928:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233489.179289:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108992:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233489.179316:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233489/real 1761233489] req@00000000db10af3c x1846780607108992/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233514 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233489.179327:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607108992/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233514 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233489.179335:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233489.179338:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233489.179341:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607108992:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233489.179346:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109056:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233489.179369:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233489/real 1761233489] req@00000000b72feabb x1846780607109056/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233514 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233489.179379:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607109056/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233514 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233489.179387:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233489.179389:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233489.179391:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109056:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:2.0:1761233489.626993:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233490.651105:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233491.674949:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233492.699058:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233493.723015:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233494.298987:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233494.299000:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233494.299004:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233494.299015:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6366 00000100:00080000:3.0:1761233494.299019:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233494.299054:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233494.299094:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607109120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233494.299124:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233494.299128:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233494.299129:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233494.299133:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6366 00000100:00080000:3.0:1761233494.299135:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233494.299138:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233494.299147:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233494.299158:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607109184/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233494.299169:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233494.299171:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233494.299173:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233494.299177:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6366 00000100:00080000:3.0:1761233494.299179:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233494.299183:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233494.299190:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233494.299207:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607109120:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761233494.299217:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607109248/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233494.299233:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233494.299236:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233494.299238:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233494.299242:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6366 00000100:00080000:3.0:1761233494.299244:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233494.299248:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233494.299254:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233494.299267:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607109312/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233494.299291:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233494/real 1761233494] req@000000006572b591 x1846780607109120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233519 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233494.299307:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607109120/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233519 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233494.299316:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233494.299318:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233494.299322:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607109120:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233494.299327:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109184:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233494.299343:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233494/real 1761233494] req@00000000fc01c41c x1846780607109184/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233519 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233494.299350:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607109184/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233519 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233494.299355:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233494.299356:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233494.299358:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109184:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233494.299378:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109248:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233494.299393:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233494/real 1761233494] req@00000000408f088c x1846780607109248/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233519 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233494.299400:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607109248/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233519 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233494.299406:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233494.299407:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233494.299409:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109248:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233494.299412:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109312:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233494.299423:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233494/real 1761233494] req@000000002ce934d3 x1846780607109312/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233519 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233494.299430:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607109312/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233519 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233494.299436:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233494.299437:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233494.299438:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109312:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233494.747015:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233495.771027:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233496.794966:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233497.819024:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233498.843011:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233499.418964:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233499.418978:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233499.418983:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233499.418995:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6371 00000100:00080000:3.0:1761233499.418999:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233499.419007:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233499.419049:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607109376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233499.419077:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233499.419080:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233499.419081:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233499.419084:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6371 00000100:00080000:3.0:1761233499.419086:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233499.419088:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233499.419096:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233499.419105:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607109440/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233499.419112:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233499.419115:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233499.419117:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233499.419119:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6371 00000100:00080000:3.0:1761233499.419121:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233499.419123:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233499.419127:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233499.419135:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607109376:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761233499.419158:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607109504/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233499.419168:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233499.419171:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233499.419172:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233499.419175:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6371 00000100:00080000:3.0:1761233499.419176:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233499.419180:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233499.419183:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233499.419194:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607109568/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233499.419255:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233499/real 1761233499] req@000000002ce934d3 x1846780607109376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233524 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233499.419295:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607109376/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233524 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233499.419308:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233499.419312:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233499.419317:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607109376:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233499.419325:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109440:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233499.419355:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233499/real 1761233499] req@00000000408f088c x1846780607109440/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233524 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233499.419364:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607109440/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233524 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233499.419371:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233499.419372:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233499.419375:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109440:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233499.419396:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109504:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233499.419413:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233499/real 1761233499] req@00000000fc01c41c x1846780607109504/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233524 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233499.419421:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607109504/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233524 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233499.419427:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233499.419428:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233499.419430:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109504:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233499.419434:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109568:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233499.419447:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233499/real 1761233499] req@000000006572b591 x1846780607109568/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233524 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233499.419460:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607109568/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233524 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233499.419466:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233499.419467:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233499.419469:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109568:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233499.867032:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233500.891011:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233501.915040:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233502.939090:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233503.963043:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233504.539088:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233504.539098:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233504.539101:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233504.539111:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6376 00000100:00080000:1.0:1761233504.539115:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233504.539121:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233504.539161:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607109632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233504.539187:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233504.539190:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233504.539192:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233504.539196:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6376 00000100:00080000:1.0:1761233504.539197:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233504.539200:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233504.539207:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233504.539216:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607109696/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233504.539224:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233504.539227:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233504.539228:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233504.539232:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6376 00000100:00080000:1.0:1761233504.539234:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233504.539237:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233504.539242:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233504.539254:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607109632:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233504.539303:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607109760/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233504.539314:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233504.539317:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233504.539318:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233504.539322:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6376 00000100:00080000:1.0:1761233504.539323:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233504.539326:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233504.539329:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233504.539338:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607109824/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233504.539406:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233504/real 1761233504] req@00000000b72feabb x1846780607109632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233529 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233504.539431:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607109632/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233529 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233504.539444:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233504.539448:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233504.539453:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607109632:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233504.539461:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109696:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233504.539490:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233504/real 1761233504] req@00000000db10af3c x1846780607109696/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233529 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233504.539500:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607109696/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233529 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233504.539506:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233504.539508:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233504.539510:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109696:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233504.539584:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109760:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233504.539610:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233504/real 1761233504] req@00000000d07717c4 x1846780607109760/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233529 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233504.539620:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607109760/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233529 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233504.539628:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233504.539630:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233504.539632:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109760:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233504.539636:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109824:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233504.539651:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233504/real 1761233504] req@00000000659d78e3 x1846780607109824/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233529 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233504.539660:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607109824/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233529 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233504.539667:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233504.539668:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233504.539670:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109824:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233504.987024:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233506.011060:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761233507.035101:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233508.059079:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233509.083168:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233509.659057:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233509.659076:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233509.659082:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233509.659098:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6382 00000100:00080000:3.0:1761233509.659103:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233509.659114:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233509.659168:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607109888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233509.659284:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233509.659289:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233509.659291:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233509.659296:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6382 00000100:00080000:3.0:1761233509.659299:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233509.659303:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233509.659315:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233509.659328:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607109952/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233509.659336:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233509.659340:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233509.659341:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233509.659346:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6382 00000100:00080000:3.0:1761233509.659348:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233509.659352:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233509.659357:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233509.659370:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607110016/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233509.659376:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607109888:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761233509.659382:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233509.659387:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233509.659388:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233509.659393:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6382 00000100:00080000:3.0:1761233509.659395:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233509.659399:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233509.659403:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233509.659415:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607110080/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233509.659529:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233509/real 1761233509] req@000000001f8e688e x1846780607109888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233534 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233509.659545:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607109888/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233534 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233509.659555:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233509.659558:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233509.659562:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607109888:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233509.659566:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109952:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233509.659584:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233509/real 1761233509] req@00000000a58ca11e x1846780607109952/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233534 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233509.659591:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607109952/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233534 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233509.659596:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233509.659597:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233509.659598:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607109952:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233509.659622:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110016:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233509.659636:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233509/real 1761233509] req@0000000083f16a6b x1846780607110016/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233534 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233509.659642:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607110016/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233534 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233509.659647:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233509.659648:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233509.659650:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110016:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233509.659653:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110080:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233509.659664:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233509/real 1761233509] req@0000000090b40c66 x1846780607110080/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233534 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233509.659671:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607110080/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233534 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233509.659676:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233509.659677:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233509.659678:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110080:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233510.107042:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233511.131056:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233512.154971:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233513.179065:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233514.203002:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233514.779069:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233514.779078:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233514.779082:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233514.779090:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6387 00000100:00080000:0.0:1761233514.779094:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233514.779099:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233514.779130:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607110144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233514.779155:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233514.779157:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233514.779158:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233514.779161:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6387 00000100:00080000:0.0:1761233514.779162:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233514.779165:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233514.779171:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233514.779179:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607110208/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233514.779185:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233514.779188:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233514.779190:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233514.779192:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6387 00000100:00080000:0.0:1761233514.779194:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233514.779196:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233514.779199:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233514.779205:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607110144:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761233514.779206:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607110272/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233514.779221:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233514.779223:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233514.779224:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233514.779226:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6387 00000100:00080000:0.0:1761233514.779227:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233514.779228:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233514.779231:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233514.779238:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607110336/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233514.779295:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233514/real 1761233514] req@00000000275aeee5 x1846780607110144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233539 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233514.779308:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607110144/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233539 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233514.779314:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233514.779316:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233514.779319:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607110144:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233514.779323:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110208:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233514.779338:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233514/real 1761233514] req@0000000001b2f813 x1846780607110208/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233539 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233514.779344:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607110208/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233539 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233514.779353:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233514.779355:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233514.779356:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110208:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233514.779390:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110272:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233514.779408:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233514/real 1761233514] req@0000000023ab3a3f x1846780607110272/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233539 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233514.779414:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607110272/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233539 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233514.779429:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233514.779431:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233514.779432:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110272:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233514.779435:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110336:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233514.779446:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233514/real 1761233514] req@00000000271a9d8e x1846780607110336/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233539 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233514.779452:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607110336/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233539 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233514.779457:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233514.779458:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233514.779459:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110336:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233515.227147:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233516.251023:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233517.275094:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233518.299157:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233519.323034:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233519.899003:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233519.899014:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233519.899017:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233519.899027:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6392 00000100:00080000:0.0:1761233519.899030:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233519.899037:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233519.899076:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607110400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233519.899103:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233519.899107:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233519.899109:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233519.899117:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6392 00000100:00080000:0.0:1761233519.899120:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233519.899123:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233519.899131:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233519.899141:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607110464/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233519.899149:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233519.899152:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233519.899153:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233519.899156:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6392 00000100:00080000:0.0:1761233519.899157:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233519.899160:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233519.899163:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233519.899172:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607110528/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233519.899179:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233519.899182:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233519.899184:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233519.899187:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6392 00000100:00080000:0.0:1761233519.899190:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233519.899194:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233519.899197:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233519.899206:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607110592/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233519.899228:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607110400:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233519.899348:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110464:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233519.899377:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110528:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233519.899412:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110592:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233520.347043:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233521.371149:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233522.395017:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761233522.971449:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233519/real 1761233522] req@00000000271a9d8e x1846780607110592/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233544 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233522.971485:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607110592/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233544 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233522.971497:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233522.971502:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233522.971506:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110592:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233522.971514:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233519/real 1761233522] req@000000002ef1f54f x1846780607110400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233544 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233522.971526:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607110400/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233544 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233522.971533:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233522.971535:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233522.971538:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607110400:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233522.971544:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233519/real 1761233522] req@0000000099514377 x1846780607110464/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233544 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233522.971561:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607110464/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233544 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233522.971568:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233522.971570:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233522.971573:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110464:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233522.971580:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233519/real 1761233522] req@000000005a096598 x1846780607110528/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233544 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233522.971590:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607110528/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233544 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233522.971598:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233522.971600:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233522.971603:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110528:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233523.418998:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233524.443033:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233525.019036:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233525.019049:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233525.019053:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233525.019064:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6397 00000100:00080000:3.0:1761233525.019068:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233525.019074:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233525.019114:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607110656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233525.019148:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233525.019152:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233525.019154:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233525.019158:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6397 00000100:00080000:3.0:1761233525.019161:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233525.019165:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233525.019177:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233525.019194:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607110720/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233525.019221:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233525.019226:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1761233525.019227:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607110656:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761233525.019229:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233525.019237:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6397 00000100:00080000:3.0:1761233525.019240:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233525.019243:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233525.019249:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233525.019264:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607110784/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233525.019274:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233525.019277:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233525.019278:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233525.019282:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6397 00000100:00080000:3.0:1761233525.019285:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233525.019288:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233525.019293:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233525.019310:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607110848/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233525.019347:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233525/real 1761233525] req@000000007361665d x1846780607110656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233550 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233525.019370:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607110656/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233550 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233525.019382:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233525.019386:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233525.019390:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607110656:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233525.019422:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110720:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233525.019453:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233525/real 1761233525] req@00000000f87d0618 x1846780607110720/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233550 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233525.019465:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607110720/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233550 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233525.019472:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233525.019474:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233525.019477:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110720:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233525.019483:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110784:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233525.019508:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233525/real 1761233525] req@00000000c7e4e432 x1846780607110784/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233550 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233525.019518:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607110784/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233550 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233525.019526:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233525.019528:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233525.019531:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110784:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233525.019536:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110848:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233525.019560:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233525/real 1761233525] req@0000000031e043f4 x1846780607110848/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233550 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233525.019571:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607110848/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233550 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233525.019580:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233525.019583:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233525.019585:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110848:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233525.467028:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233526.491028:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233527.515024:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233528.538938:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233529.563063:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233530.138987:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233530.139000:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233530.139005:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233530.139017:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6402 00000100:00080000:0.0:1761233530.139021:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233530.139029:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233530.139072:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607110912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233530.139101:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233530.139105:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233530.139106:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233530.139109:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6402 00000100:00080000:0.0:1761233530.139111:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233530.139114:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233530.139140:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233530.139152:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607110976/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233530.139160:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233530.139163:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233530.139165:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233530.139170:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6402 00000100:00080000:0.0:1761233530.139171:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233530.139174:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233530.139178:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233530.139187:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607111040/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233530.139194:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233530.139197:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233530.139198:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233530.139200:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6402 00000100:00080000:0.0:1761233530.139202:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233530.139204:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233530.139207:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233530.139214:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607111104/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233530.139225:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607110912:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233530.139315:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233530/real 1761233530] req@0000000023ab3a3f x1846780607110912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233555 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233530.139329:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607110912/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233555 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233530.139336:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233530.139339:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233530.139342:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607110912:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233530.139346:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110976:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233530.139361:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233530/real 1761233530] req@0000000001b2f813 x1846780607110976/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233555 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233530.139368:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607110976/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233555 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233530.139372:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233530.139373:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233530.139375:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607110976:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233530.139378:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111040:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233530.139390:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233530/real 1761233530] req@00000000275aeee5 x1846780607111040/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233555 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233530.139397:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607111040/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233555 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233530.139402:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233530.139403:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233530.139404:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111040:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233530.139425:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111104:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233530.139437:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233530/real 1761233530] req@000000000243251f x1846780607111104/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233555 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233530.139443:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607111104/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233555 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233530.139448:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233530.139449:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233530.139450:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111104:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233530.586979:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233531.611046:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233532.635012:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233533.659005:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233534.683067:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233535.259037:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233535.259060:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233535.259065:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233535.259077:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6407 00000100:00080000:3.0:1761233535.259081:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233535.259088:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233535.259134:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607111168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233535.259163:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233535.259166:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233535.259167:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233535.259206:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6407 00000100:00080000:3.0:1761233535.259208:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233535.259214:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1761233535.259222:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607111168:192.168.201.104@tcp:250:kworker/u8:1.0 00000008:00100000:3.0:1761233535.259228:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233535.259245:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607111232/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233535.259259:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233535.259262:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233535.259265:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233535.259268:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6407 00000100:00080000:3.0:1761233535.259270:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233535.259273:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233535.259276:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233535.259286:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607111296/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233535.259292:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233535.259295:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233535.259296:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233535.259298:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6407 00000100:00080000:3.0:1761233535.259299:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233535.259302:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233535.259304:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233535.259310:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607111360/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233535.259334:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233535/real 1761233535] req@0000000086ccb527 x1846780607111168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233560 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233535.259359:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607111168/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233560 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233535.259369:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233535.259373:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233535.259378:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607111168:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233535.259407:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111232:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233535.259437:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233535/real 1761233535] req@00000000bd684ccc x1846780607111232/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233560 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233535.259449:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607111232/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233560 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233535.259457:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233535.259459:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233535.259462:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111232:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233535.259469:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111296:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233535.259495:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233535/real 1761233535] req@0000000018a8f5da x1846780607111296/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233560 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233535.259506:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607111296/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233560 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233535.259514:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233535.259517:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233535.259520:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111296:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233535.259525:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111360:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233535.259549:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233535/real 1761233535] req@000000005a9bdefd x1846780607111360/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233560 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233535.259559:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607111360/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233560 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233535.259568:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233535.259570:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233535.259573:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111360:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233535.707007:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233536.731112:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233537.755050:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233538.779110:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233539.803184:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233540.378946:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233540.378955:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233540.378958:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233540.378989:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6412 00000100:00080000:3.0:1761233540.378992:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233540.378997:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233540.379028:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607111424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233540.379050:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233540.379052:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233540.379054:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233540.379056:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6412 00000100:00080000:3.0:1761233540.379057:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233540.379059:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233540.379065:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233540.379072:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607111488/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233540.379077:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233540.379079:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233540.379081:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233540.379083:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6412 00000100:00080000:3.0:1761233540.379084:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233540.379086:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233540.379088:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233540.379096:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607111552/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233540.379101:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233540.379103:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233540.379104:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233540.379106:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6412 00000100:00100000:2.0:1761233540.379116:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607111424:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761233540.379149:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233540.379160:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233540.379163:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233540.379174:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607111616/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233540.379216:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233540/real 1761233540] req@00000000daa39bb1 x1846780607111424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233565 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233540.379235:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607111424/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233565 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233540.379245:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233540.379249:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233540.379252:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607111424:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233540.379261:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111488:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233540.379291:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233540/real 1761233540] req@000000003ad149cc x1846780607111488/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233565 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233540.379303:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607111488/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233565 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233540.379311:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233540.379314:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233540.379317:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111488:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233540.379323:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111552:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233540.379351:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233540/real 1761233540] req@0000000088473b3e x1846780607111552/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233565 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233540.379362:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607111552/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233565 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233540.379371:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233540.379373:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233540.379376:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111552:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233540.379405:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111616:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233540.379434:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233540/real 1761233540] req@000000004e3f612a x1846780607111616/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233565 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233540.379445:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607111616/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233565 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233540.379453:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233540.379456:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233540.379459:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111616:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233540.827042:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233541.851051:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233542.875083:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233543.898995:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233544.923051:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233545.498942:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233545.498951:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233545.498954:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233545.498964:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6417 00000100:00080000:3.0:1761233545.498967:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233545.498972:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233545.499011:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607111680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233545.499033:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233545.499036:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233545.499037:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233545.499039:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6417 00000100:00080000:3.0:1761233545.499041:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233545.499043:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233545.499049:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233545.499059:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607111744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233545.499066:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233545.499069:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233545.499069:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233545.499072:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6417 00000100:00080000:3.0:1761233545.499073:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233545.499075:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233545.499079:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233545.499083:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607111680:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761233545.499088:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607111808/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233545.499096:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233545.499098:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233545.499099:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233545.499101:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6417 00000100:00080000:3.0:1761233545.499102:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233545.499104:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233545.499106:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233545.499113:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607111872/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233545.499159:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233545/real 1761233545] req@000000004e3f612a x1846780607111680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233570 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233545.499172:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607111680/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233570 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233545.499178:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233545.499180:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233545.499183:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607111680:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233545.499188:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111744:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233545.499203:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233545/real 1761233545] req@0000000088473b3e x1846780607111744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233570 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233545.499210:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607111744/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233570 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233545.499214:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233545.499215:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233545.499216:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111744:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233545.499235:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111808:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233545.499247:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233545/real 1761233545] req@000000003ad149cc x1846780607111808/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233570 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233545.499253:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607111808/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233570 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233545.499258:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233545.499259:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233545.499261:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111808:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233545.499264:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111872:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233545.499274:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233545/real 1761233545] req@00000000daa39bb1 x1846780607111872/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233570 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233545.499284:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607111872/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233570 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233545.499289:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233545.499290:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233545.499291:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607111872:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233545.947005:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233546.971094:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233547.995011:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233549.019042:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233550.043070:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233550.618968:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233550.618976:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233550.618979:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233550.618988:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6423 00000100:00080000:0.0:1761233550.618990:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233550.618996:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233550.619027:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607111936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233550.619047:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233550.619050:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233550.619051:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233550.619053:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6423 00000100:00080000:0.0:1761233550.619055:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233550.619057:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233550.619063:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233550.619070:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607112000/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233550.619076:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233550.619078:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233550.619079:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233550.619082:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6423 00000100:00080000:0.0:1761233550.619084:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233550.619085:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233550.619088:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233550.619094:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607112064/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233550.619100:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233550.619101:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233550.619103:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761233550.619104:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607111936:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761233550.619108:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6423 00000100:00080000:0.0:1761233550.619112:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233550.619114:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233550.619116:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233550.619124:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607112128/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233550.619182:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233550/real 1761233550] req@00000000b8f2be78 x1846780607111936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233575 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233550.619195:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607111936/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233575 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233550.619202:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233550.619205:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233550.619207:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607111936:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233550.619211:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112000:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233550.619225:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233550/real 1761233550] req@00000000fcce404c x1846780607112000/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233575 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233550.619232:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607112000/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233575 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233550.619236:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233550.619237:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233550.619238:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112000:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233550.619257:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112064:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233550.619270:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233550/real 1761233550] req@000000007f60e210 x1846780607112064/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233575 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233550.619277:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607112064/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233575 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233550.619282:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233550.619283:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233550.619284:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112064:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233550.619287:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112128:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233550.619298:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233550/real 1761233550] req@000000000243251f x1846780607112128/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233575 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233550.619304:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607112128/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233575 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233550.619309:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233550.619310:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233550.619311:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112128:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233551.066971:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233552.091041:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233553.115061:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233554.139075:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233555.163026:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233555.738911:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233555.738921:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233555.738924:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233555.738932:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6428 00000100:00080000:0.0:1761233555.738934:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233555.738940:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233555.738968:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607112192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233555.738991:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233555.738995:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233555.738996:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233555.739001:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6428 00000100:00080000:0.0:1761233555.739003:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233555.739006:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233555.739014:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233555.739029:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607112256/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233555.739039:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607112192:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761233555.739042:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233555.739048:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233555.739049:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233555.739053:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6428 00000100:00080000:0.0:1761233555.739055:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233555.739057:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233555.739063:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233555.739072:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607112320/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233555.739078:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233555.739081:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233555.739081:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233555.739084:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6428 00000100:00080000:0.0:1761233555.739085:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233555.739087:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233555.739089:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233555.739095:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607112384/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233555.739122:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233555/real 1761233555] req@00000000275aeee5 x1846780607112192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233580 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233555.739138:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607112192/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233580 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233555.739149:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233555.739152:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233555.739156:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607112192:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233555.739179:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112256:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233555.739210:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233555/real 1761233555] req@0000000001b2f813 x1846780607112256/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233580 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233555.739222:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607112256/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233580 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233555.739230:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233555.739233:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233555.739236:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112256:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233555.739241:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112320:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233555.739260:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233555/real 1761233555] req@0000000023ab3a3f x1846780607112320/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233580 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233555.739271:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607112320/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233580 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233555.739279:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233555.739281:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233555.739284:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112320:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233555.739289:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112384:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233555.739306:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233555/real 1761233555] req@000000005a096598 x1846780607112384/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233580 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233555.739316:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607112384/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233580 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233555.739324:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233555.739326:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233555.739328:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112384:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233556.187031:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233557.211023:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233558.235055:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233559.258998:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233560.283036:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233560.858967:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233560.858975:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233560.858978:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233560.858987:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6433 00000100:00080000:3.0:1761233560.858989:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233560.858995:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233560.859025:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607112448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233560.859046:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233560.859049:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233560.859050:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233560.859053:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6433 00000100:00080000:3.0:1761233560.859054:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233560.859056:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233560.859062:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233560.859069:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607112512/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233560.859074:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233560.859076:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233560.859077:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233560.859079:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6433 00000100:00080000:3.0:1761233560.859081:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233560.859083:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233560.859085:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233560.859094:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607112576/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233560.859099:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233560.859101:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233560.859102:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233560.859104:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6433 00000100:00080000:3.0:1761233560.859105:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233560.859107:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1761233560.859111:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607112448:192.168.201.104@tcp:250:kworker/u8:2.0 00000008:00100000:3.0:1761233560.859112:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233560.859121:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607112640/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233560.859188:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233560/real 1761233560] req@000000005a9bdefd x1846780607112448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233585 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233560.859201:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607112448/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233585 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233560.859208:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233560.859210:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233560.859213:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607112448:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233560.859217:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112512:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233560.859231:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233560/real 1761233560] req@0000000018a8f5da x1846780607112512/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233585 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233560.859238:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607112512/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233585 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233560.859242:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233560.859244:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233560.859246:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112512:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233560.859249:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112576:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233560.859260:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233560/real 1761233560] req@00000000bd684ccc x1846780607112576/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233585 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233560.859266:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607112576/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233585 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233560.859270:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233560.859271:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233560.859273:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112576:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233560.859292:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112640:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233560.859303:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233560/real 1761233560] req@0000000086ccb527 x1846780607112640/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233585 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233560.859310:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607112640/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233585 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233560.859314:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233560.859315:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233560.859317:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112640:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233561.307010:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233562.331007:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233563.355007:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233564.379041:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233565.403022:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233565.978984:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233565.978993:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233565.978996:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233565.979005:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6438 00000100:00080000:1.0:1761233565.979007:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233565.979012:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233565.979048:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607112704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233565.979069:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233565.979072:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233565.979073:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233565.979076:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6438 00000100:00080000:1.0:1761233565.979079:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233565.979081:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233565.979087:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233565.979095:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607112768/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233565.979100:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233565.979103:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233565.979103:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233565.979105:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6438 00000100:00080000:1.0:1761233565.979106:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233565.979108:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233565.979110:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233565.979115:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607112832/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233565.979120:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233565.979121:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233565.979122:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233565.979124:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6438 00000100:00080000:1.0:1761233565.979125:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233565.979127:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233565.979130:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233565.979135:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607112896/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233565.979159:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607112704:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233565.979265:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233565/real 1761233565] req@00000000659d78e3 x1846780607112704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233590 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233565.979282:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607112704/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233590 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233565.979289:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233565.979291:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233565.979294:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607112704:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233565.979298:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112768:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233565.979314:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233565/real 1761233565] req@00000000d07717c4 x1846780607112768/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233590 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233565.979321:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607112768/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233590 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233565.979325:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233565.979326:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233565.979328:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112768:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233565.979334:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112832:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233565.979356:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233565/real 1761233565] req@00000000db10af3c x1846780607112832/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233590 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233565.979367:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607112832/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233590 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233565.979376:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233565.979378:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233565.979381:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112832:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233565.979404:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112896:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233565.979426:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233565/real 1761233565] req@00000000b72feabb x1846780607112896/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233590 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233565.979433:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607112896/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233590 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233565.979438:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233565.979439:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233565.979441:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607112896:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233566.427020:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233567.451041:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233568.475044:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233569.499029:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233570.523035:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233571.099031:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233571.099041:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233571.099045:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233571.099055:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6443 00000100:00080000:3.0:1761233571.099058:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233571.099064:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233571.099099:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607112960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233571.099126:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233571.099129:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233571.099130:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233571.099133:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6443 00000100:00080000:3.0:1761233571.099135:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233571.099137:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233571.099144:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233571.099152:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607113024/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233571.099158:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233571.099161:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233571.099162:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233571.099164:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6443 00000100:00080000:3.0:1761233571.099165:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233571.099167:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233571.099170:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233571.099179:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607113088/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233571.099185:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233571.099187:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233571.099189:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233571.099192:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6443 00000100:00080000:3.0:1761233571.099194:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233571.099196:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233571.099199:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233571.099205:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607113152/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233571.099215:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607112960:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233571.099297:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233571/real 1761233571] req@0000000031e043f4 x1846780607112960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233596 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233571.099311:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607112960/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233596 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233571.099319:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233571.099322:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233571.099325:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607112960:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233571.099330:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113024:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233571.099346:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233571/real 1761233571] req@00000000c7e4e432 x1846780607113024/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233596 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233571.099354:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607113024/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233596 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233571.099364:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233571.099366:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233571.099368:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113024:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233571.099371:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113088:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233571.099384:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233571/real 1761233571] req@00000000f87d0618 x1846780607113088/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233596 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233571.099391:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607113088/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233596 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233571.099396:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233571.099397:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233571.099399:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113088:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233571.099420:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113152:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233571.099433:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233571/real 1761233571] req@000000007361665d x1846780607113152/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233596 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233571.099440:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607113152/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233596 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233571.099445:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233571.099447:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233571.099449:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113152:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233571.547062:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233572.571003:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233573.595070:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233574.619015:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233575.643134:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233576.218934:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233576.218947:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233576.218951:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233576.218963:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6448 00000100:00080000:3.0:1761233576.218967:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233576.218975:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233576.219018:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607113216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233576.219055:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233576.219060:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233576.219063:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233576.219069:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6448 00000100:00080000:3.0:1761233576.219071:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233576.219075:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233576.219086:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233576.219104:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607113280/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233576.219115:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233576.219119:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233576.219121:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233576.219126:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6448 00000100:00080000:3.0:1761233576.219129:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233576.219133:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1761233576.219136:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607113216:192.168.201.104@tcp:250:kworker/u8:2.0 00000008:00100000:3.0:1761233576.219142:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233576.219163:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607113344/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233576.219175:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233576.219179:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233576.219181:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233576.219185:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6448 00000100:00080000:3.0:1761233576.219188:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233576.219192:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233576.219198:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233576.219214:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607113408/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233576.219216:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233576/real 1761233576] req@0000000090b40c66 x1846780607113216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233601 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233576.219241:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607113216/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233601 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233576.219249:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233576.219251:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233576.219254:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607113216:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233576.219259:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113280:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233576.219276:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233576/real 1761233576] req@0000000083f16a6b x1846780607113280/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233601 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233576.219283:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607113280/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233601 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233576.219289:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233576.219290:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233576.219292:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113280:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233576.219311:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113344:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233576.219325:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233576/real 1761233576] req@00000000a58ca11e x1846780607113344/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233601 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233576.219332:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607113344/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233601 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233576.219337:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233576.219339:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233576.219340:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113344:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233576.219343:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113408:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233576.219354:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233576/real 1761233576] req@000000001f8e688e x1846780607113408/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233601 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233576.219359:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607113408/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233601 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233576.219364:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233576.219365:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233576.219367:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113408:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233576.667061:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233577.691198:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233578.715140:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233579.739124:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233580.763097:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233581.338995:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233581.339008:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233581.339013:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233581.339025:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6453 00000100:00080000:3.0:1761233581.339030:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233581.339039:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233581.339087:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607113472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233581.339128:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233581.339132:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233581.339134:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233581.339139:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6453 00000100:00080000:3.0:1761233581.339142:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233581.339145:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233581.339155:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233581.339168:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607113536/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233581.339178:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233581.339182:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233581.339183:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233581.339188:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6453 00000100:00080000:3.0:1761233581.339191:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233581.339195:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233581.339199:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233581.339211:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607113600/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233581.339220:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233581.339224:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233581.339226:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233581.339230:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6453 00000100:00080000:3.0:1761233581.339233:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233581.339238:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233581.339242:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233581.339252:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607113664/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233581.339275:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607113472:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233581.339419:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233581/real 1761233581] req@000000006572b591 x1846780607113472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233606 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233581.339441:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607113472/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233606 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233581.339453:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233581.339457:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233581.339461:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607113472:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233581.339470:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113536:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233581.339497:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233581/real 1761233581] req@00000000fc01c41c x1846780607113536/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233606 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233581.339519:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607113536/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233606 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233581.339527:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233581.339530:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233581.339533:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113536:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233581.339539:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113600:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233581.339560:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233581/real 1761233581] req@00000000408f088c x1846780607113600/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233606 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233581.339572:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607113600/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233606 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233581.339581:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233581.339584:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233581.339586:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113600:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233581.339618:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113664:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233581.339640:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233581/real 1761233581] req@000000002ce934d3 x1846780607113664/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233606 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233581.339651:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607113664/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233606 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233581.339660:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233581.339662:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233581.339666:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113664:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233581.787079:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233582.811039:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233583.835020:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233584.858966:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233585.883035:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233586.458953:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233586.458967:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233586.458972:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233586.458984:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6458 00000100:00080000:3.0:1761233586.458989:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233586.458998:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233586.459047:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607113728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233586.459081:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233586.459086:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233586.459087:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233586.459092:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6458 00000100:00080000:3.0:1761233586.459094:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233586.459097:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233586.459107:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233586.459119:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607113792/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233586.459130:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233586.459133:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233586.459135:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233586.459139:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6458 00000100:00080000:3.0:1761233586.459141:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233586.459144:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233586.459147:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233586.459157:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607113856/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233586.459167:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233586.459171:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233586.459173:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233586.459178:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6458 00000100:00080000:3.0:1761233586.459181:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233586.459186:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233586.459190:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233586.459201:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607113920/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233586.459212:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607113728:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233586.459342:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113792:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233586.459372:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113856:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233586.459407:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113920:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233586.907144:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233587.931164:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233588.955090:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761233589.531226:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233586/real 1761233589] req@000000006572b591 x1846780607113920/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233611 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233589.531254:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607113920/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233611 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233589.531266:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233589.531270:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233589.531274:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113920:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233589.531280:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233586/real 1761233589] req@000000002ce934d3 x1846780607113728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233611 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233589.531293:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607113728/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233611 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233589.531300:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233589.531302:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233589.531305:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607113728:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233589.531311:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233586/real 1761233589] req@00000000408f088c x1846780607113792/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233611 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233589.531329:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607113792/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233611 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233589.531335:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233589.531338:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233589.531340:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113792:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233589.531346:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233586/real 1761233589] req@00000000fc01c41c x1846780607113856/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233611 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233589.531355:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607113856/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233611 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233589.531362:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233589.531364:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233589.531367:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607113856:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233589.979070:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233591.003077:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233591.579094:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233591.579109:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233591.579113:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233591.579125:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6463 00000100:00080000:0.0:1761233591.579130:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233591.579138:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233591.579230:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607113984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233591.579262:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233591.579266:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233591.579268:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233591.579272:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6463 00000100:00080000:0.0:1761233591.579275:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233591.579279:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233591.579287:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233591.579300:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607114048/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233591.579309:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233591.579312:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233591.579314:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233591.579319:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6463 00000100:00080000:0.0:1761233591.579321:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233591.579325:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233591.579330:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233591.579339:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607113984:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:0.0:1761233591.579342:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607114112/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233591.579356:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233591.579359:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233591.579360:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233591.579363:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6463 00000100:00080000:0.0:1761233591.579366:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233591.579369:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233591.579373:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233591.579384:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607114176/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233591.579454:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233591/real 1761233591] req@0000000099514377 x1846780607113984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233616 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233591.579474:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607113984/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233616 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233591.579485:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233591.579488:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233591.579494:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607113984:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233591.579501:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114048:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233591.579531:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233591/real 1761233591] req@000000002ef1f54f x1846780607114048/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233616 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233591.579542:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607114048/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233616 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233591.579549:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233591.579551:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233591.579553:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114048:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233591.579579:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114112:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233591.579604:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233591/real 1761233591] req@00000000271a9d8e x1846780607114112/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233616 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233591.579614:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607114112/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233616 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233591.579622:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233591.579624:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233591.579627:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114112:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233591.579633:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114176:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233591.579656:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233591/real 1761233591] req@000000005a096598 x1846780607114176/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233616 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233591.579665:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607114176/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233616 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233591.579673:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233591.579676:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233591.579679:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114176:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233592.027103:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233593.051035:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233594.075114:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233595.099033:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233596.122945:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233596.699048:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233596.699058:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233596.699061:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233596.699070:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6469 00000100:00080000:0.0:1761233596.699073:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233596.699093:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233596.699126:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607114240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233596.699142:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233596.699145:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233596.699146:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233596.699149:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6469 00000100:00080000:0.0:1761233596.699151:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233596.699153:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233596.699160:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233596.699168:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607114304/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233596.699175:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607114240:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:0.0:1761233596.699176:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233596.699180:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233596.699181:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233596.699187:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6469 00000100:00080000:0.0:1761233596.699188:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233596.699190:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233596.699194:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233596.699203:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607114368/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233596.699209:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233596.699211:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233596.699212:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233596.699214:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6469 00000100:00080000:0.0:1761233596.699215:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233596.699217:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233596.699220:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233596.699226:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607114432/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233596.699270:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233596/real 1761233596] req@0000000023ab3a3f x1846780607114240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233621 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233596.699288:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607114240/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233621 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233596.699298:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233596.699301:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233596.699305:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607114240:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233596.699329:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114304:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233596.699358:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233596/real 1761233596] req@0000000001b2f813 x1846780607114304/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233621 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233596.699370:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607114304/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233621 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233596.699377:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233596.699379:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233596.699382:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114304:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233596.699388:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114368:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233596.699413:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233596/real 1761233596] req@00000000275aeee5 x1846780607114368/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233621 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233596.699430:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607114368/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233621 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233596.699438:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233596.699441:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233596.699444:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114368:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233596.699449:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114432:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233596.699490:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233596/real 1761233596] req@000000000243251f x1846780607114432/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233621 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233596.699501:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607114432/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233621 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233596.699509:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233596.699512:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233596.699515:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114432:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233597.147035:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233598.171028:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233599.194999:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233600.219044:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233601.243047:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233601.818963:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233601.818972:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233601.818975:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233601.818984:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6474 00000100:00080000:3.0:1761233601.818987:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233601.818992:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233601.819044:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607114496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233601.819070:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233601.819072:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233601.819073:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233601.819076:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6474 00000100:00080000:3.0:1761233601.819078:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233601.819080:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233601.819086:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233601.819093:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607114560/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233601.819098:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233601.819100:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233601.819100:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233601.819103:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6474 00000100:00080000:3.0:1761233601.819104:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233601.819106:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233601.819108:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233601.819116:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607114624/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233601.819120:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233601.819122:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233601.819123:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233601.819126:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6474 00000100:00080000:3.0:1761233601.819127:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233601.819130:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233601.819134:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233601.819138:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607114496:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:3.0:1761233601.819142:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607114688/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233601.819222:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233601/real 1761233601] req@000000001f8e688e x1846780607114496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233626 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233601.819236:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607114496/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233626 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233601.819254:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233601.819256:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233601.819259:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607114496:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233601.819264:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114560:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233601.819279:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233601/real 1761233601] req@00000000a58ca11e x1846780607114560/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233626 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233601.819285:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607114560/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233626 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233601.819289:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233601.819290:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233601.819292:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114560:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233601.819295:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114624:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233601.819306:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233601/real 1761233601] req@0000000083f16a6b x1846780607114624/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233626 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233601.819312:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607114624/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233626 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233601.819318:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233601.819319:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233601.819320:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114624:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233601.819339:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114688:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233601.819350:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233601/real 1761233601] req@0000000090b40c66 x1846780607114688/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233626 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233601.819356:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607114688/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233626 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233601.819361:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233601.819362:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233601.819364:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114688:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233602.267051:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233603.291072:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233604.315078:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233605.339049:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233606.363134:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233606.939071:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233606.939085:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233606.939089:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233606.939102:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6479 00000100:00080000:3.0:1761233606.939107:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233606.939116:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233606.939162:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607114752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233606.939200:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233606.939204:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233606.939206:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233606.939211:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6479 00000100:00080000:3.0:1761233606.939214:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233606.939217:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233606.939227:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233606.939239:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607114816/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233606.939257:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233606.939261:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233606.939263:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233606.939266:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6479 00000100:00080000:3.0:1761233606.939268:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233606.939271:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233606.939275:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233606.939287:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607114880/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233606.939297:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233606.939301:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233606.939303:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233606.939308:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6479 00000100:00080000:3.0:1761233606.939310:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233606.939314:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233606.939318:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233606.939329:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607114944/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233606.939343:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607114752:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233606.939456:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233606/real 1761233606] req@000000007361665d x1846780607114752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233631 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233606.939478:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607114752/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233631 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233606.939490:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233606.939494:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233606.939499:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607114752:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233606.939507:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114816:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233606.939534:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233606/real 1761233606] req@00000000f87d0618 x1846780607114816/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233631 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233606.939546:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607114816/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233631 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233606.939554:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233606.939557:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233606.939560:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114816:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233606.939565:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114880:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233606.939588:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233606/real 1761233606] req@00000000c7e4e432 x1846780607114880/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233631 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233606.939600:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607114880/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233631 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233606.939609:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233606.939611:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233606.939614:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114880:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233606.939645:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114944:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233606.939668:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233606/real 1761233606] req@0000000031e043f4 x1846780607114944/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233631 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233606.939679:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607114944/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233631 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233606.939687:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233606.939689:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233606.939693:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607114944:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233607.387090:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233608.411076:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233609.435105:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233610.459011:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233611.483106:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233612.058970:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233612.058978:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233612.058981:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233612.058989:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6484 00000100:00080000:1.0:1761233612.058992:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233612.058998:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233612.059030:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607115008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233612.059052:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233612.059055:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233612.059056:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233612.059059:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6484 00000100:00080000:1.0:1761233612.059060:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233612.059062:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233612.059068:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233612.059077:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607115072/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233612.059083:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233612.059085:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233612.059087:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233612.059089:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6484 00000100:00080000:1.0:1761233612.059091:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233612.059093:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233612.059096:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233612.059102:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607115136/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233612.059108:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607115008:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761233612.059111:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233612.059115:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233612.059116:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233612.059118:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6484 00000100:00080000:1.0:1761233612.059119:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233612.059121:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233612.059124:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233612.059132:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607115200/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233612.059183:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233612/real 1761233612] req@00000000b72feabb x1846780607115008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233637 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233612.059196:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607115008/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233637 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233612.059203:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233612.059206:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233612.059209:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607115008:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233612.059213:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115072:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233612.059227:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233612/real 1761233612] req@00000000db10af3c x1846780607115072/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233637 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233612.059234:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607115072/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233637 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233612.059238:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233612.059239:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233612.059241:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115072:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233612.059256:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115136:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233612.059269:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233612/real 1761233612] req@00000000d07717c4 x1846780607115136/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233637 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233612.059274:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607115136/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233637 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233612.059279:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233612.059280:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233612.059282:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115136:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233612.059285:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115200:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233612.059295:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233612/real 1761233612] req@00000000659d78e3 x1846780607115200/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233637 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233612.059305:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607115200/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233637 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233612.059310:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233612.059311:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233612.059312:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115200:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233612.507029:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233613.531026:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233614.555035:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233615.579006:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233616.603143:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233617.179005:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233617.179016:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233617.179020:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233617.179030:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6489 00000100:00080000:0.0:1761233617.179034:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233617.179041:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233617.179076:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607115264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233617.179106:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233617.179110:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233617.179112:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233617.179116:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6489 00000100:00080000:0.0:1761233617.179118:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233617.179122:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233617.179133:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233617.179148:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607115328/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233617.179157:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607115264:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761233617.179160:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233617.179165:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233617.179166:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233617.179171:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6489 00000100:00080000:0.0:1761233617.179174:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233617.179177:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233617.179183:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233617.179194:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607115392/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233617.179200:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233617.179202:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233617.179203:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233617.179205:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6489 00000100:00080000:0.0:1761233617.179207:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233617.179209:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233617.179212:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233617.179218:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607115456/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233617.179237:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233617/real 1761233617] req@000000007f60e210 x1846780607115264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233642 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233617.179251:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607115264/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233642 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233617.179258:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233617.179260:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233617.179263:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607115264:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233617.179280:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115328:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233617.179296:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233617/real 1761233617] req@00000000fcce404c x1846780607115328/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233642 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233617.179303:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607115328/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233642 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233617.179307:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233617.179309:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233617.179310:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115328:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233617.179314:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115392:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233617.179326:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233617/real 1761233617] req@00000000b8f2be78 x1846780607115392/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233642 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233617.179332:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607115392/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233642 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233617.179337:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233617.179338:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233617.179340:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115392:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233617.179343:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115456:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233617.179353:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233617/real 1761233617] req@000000000243251f x1846780607115456/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233642 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233617.179359:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607115456/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233642 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233617.179364:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233617.179365:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233617.179366:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115456:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233617.627030:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233618.650992:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233619.674982:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233620.699034:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233621.723090:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233622.298939:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233622.298952:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233622.298956:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233622.298969:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6494 00000100:00080000:0.0:1761233622.298973:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233622.298980:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233622.299018:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607115520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233622.299048:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233622.299058:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233622.299060:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233622.299064:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6494 00000100:00080000:0.0:1761233622.299067:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233622.299071:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233622.299081:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233622.299095:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607115584/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233622.299101:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607115520:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761233622.299117:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233622.299121:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233622.299123:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233622.299129:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6494 00000100:00080000:0.0:1761233622.299132:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233622.299136:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233622.299161:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233622.299178:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607115648/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233622.299194:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233622/real 1761233622] req@00000000275aeee5 x1846780607115520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233647 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:0.0:1761233622.299195:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233622.299204:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233622.299206:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233622.299215:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6494 00000100:00080000:0.0:1761233622.299218:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233622.299224:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000100:00100000:2.0:1761233622.299229:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607115520/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233647 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000008:00100000:0.0:1761233622.299230:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00080000:2.0:1761233622.299246:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1761233622.299248:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607115712/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233622.299253:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233622.299263:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607115520:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233622.299294:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115584:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233622.299328:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233622/real 1761233622] req@0000000001b2f813 x1846780607115584/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233647 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233622.299338:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607115584/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233647 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233622.299345:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233622.299347:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233622.299351:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115584:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233622.299357:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115648:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233622.299371:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233622/real 1761233622] req@0000000023ab3a3f x1846780607115648/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233647 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233622.299379:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607115648/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233647 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233622.299385:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233622.299386:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233622.299388:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115648:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233622.299392:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115712:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233622.299405:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233622/real 1761233622] req@000000005a096598 x1846780607115712/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233647 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233622.299412:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607115712/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233647 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233622.299418:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233622.299419:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233622.299421:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115712:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233622.747027:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233623.771048:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233624.795166:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233625.819095:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233626.843126:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233627.418995:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233627.419007:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233627.419012:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233627.419024:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6499 00000100:00080000:0.0:1761233627.419028:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233627.419035:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233627.419078:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000271a9d8e x1846780607115776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233627.419112:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233627.419117:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233627.419119:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233627.419124:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6499 00000100:00080000:0.0:1761233627.419126:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233627.419129:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233627.419138:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233627.419152:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ef1f54f x1846780607115840/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233627.419162:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233627.419166:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233627.419167:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233627.419171:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6499 00000100:00080000:0.0:1761233627.419172:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233627.419175:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233627.419178:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233627.419187:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099514377 x1846780607115904/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233627.419194:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233627.419197:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233627.419198:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233627.419201:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6499 00000100:00080000:0.0:1761233627.419203:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233627.419206:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233627.419211:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233627.419220:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607115968/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233627.419260:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607115776:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233627.419372:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233627/real 1761233627] req@00000000271a9d8e x1846780607115776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233652 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233627.419391:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000271a9d8e x1846780607115776/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233652 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233627.419402:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233627.419405:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233627.419410:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000271a9d8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607115776:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233627.419416:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115840:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233627.419439:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233627/real 1761233627] req@000000002ef1f54f x1846780607115840/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233652 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233627.419450:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ef1f54f x1846780607115840/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233652 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233627.419462:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233627.419464:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233627.419467:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef1f54f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115840:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233627.419472:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115904:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233627.419491:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233627/real 1761233627] req@0000000099514377 x1846780607115904/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233652 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233627.419500:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000099514377 x1846780607115904/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233652 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233627.419508:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233627.419509:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233627.419512:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099514377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115904:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233627.419517:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115968:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233627.419534:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233627/real 1761233627] req@000000005a096598 x1846780607115968/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233652 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233627.419544:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607115968/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233652 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233627.419552:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233627.419554:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233627.419556:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607115968:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233627.867252:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233628.891014:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233629.914970:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233630.939093:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233631.963063:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233632.539022:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233632.539038:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233632.539043:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233632.539055:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6504 00000100:00080000:3.0:1761233632.539058:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233632.539066:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233632.539109:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607116032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233632.539140:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233632.539144:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233632.539146:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233632.539150:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6504 00000100:00080000:3.0:1761233632.539152:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233632.539158:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233632.539169:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233632.539183:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607116096/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233632.539195:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233632.539198:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233632.539200:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233632.539205:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6504 00000100:00100000:2.0:1761233632.539212:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607116032:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761233632.539212:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233632.539223:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233632.539234:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233632.539253:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607116160/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233632.539262:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233632.539265:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233632.539267:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233632.539271:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6504 00000100:00080000:3.0:1761233632.539273:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233632.539277:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233632.539282:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233632.539290:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607116224/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233632.539344:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233632/real 1761233632] req@0000000086ccb527 x1846780607116032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233657 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233632.539365:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607116032/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233657 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233632.539376:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233632.539380:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233632.539384:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607116032:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233632.539411:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116096:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233632.539443:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233632/real 1761233632] req@00000000bd684ccc x1846780607116096/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233657 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233632.539455:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607116096/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233657 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233632.539462:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233632.539465:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233632.539468:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116096:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233632.539485:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116160:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233632.539531:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233632/real 1761233632] req@0000000018a8f5da x1846780607116160/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233657 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233632.539543:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607116160/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233657 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233632.539557:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233632.539560:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233632.539563:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116160:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233632.539569:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116224:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233632.539594:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233632/real 1761233632] req@000000005a9bdefd x1846780607116224/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233657 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233632.539604:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607116224/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233657 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233632.539612:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233632.539614:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233632.539617:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116224:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233632.987027:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233634.011018:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233635.035056:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233636.059060:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233637.083118:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233637.659154:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233637.659166:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233637.659170:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233637.659182:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6510 00000100:00080000:1.0:1761233637.659186:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233637.659193:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233637.659236:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607116288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233637.659272:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233637.659276:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233637.659278:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233637.659282:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6510 00000100:00080000:1.0:1761233637.659284:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233637.659287:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233637.659299:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233637.659312:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607116352/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233637.659321:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233637.659324:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233637.659326:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233637.659329:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6510 00000100:00080000:1.0:1761233637.659332:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233637.659335:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233637.659339:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233637.659349:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607116416/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233637.659362:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607116288:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761233637.659364:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233637.659368:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233637.659370:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233637.659386:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6510 00000100:00080000:1.0:1761233637.659389:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233637.659394:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233637.659403:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233637.659426:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607116480/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233637.659561:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233637/real 1761233637] req@00000000659d78e3 x1846780607116288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233662 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233637.659589:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607116288/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233662 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233637.659607:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233637.659612:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233637.659619:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607116288:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233637.659632:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116352:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233637.659684:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233637/real 1761233637] req@00000000d07717c4 x1846780607116352/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233662 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233637.659699:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607116352/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233662 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233637.659707:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233637.659710:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233637.659713:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116352:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233637.659747:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116416:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233637.659775:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233637/real 1761233637] req@00000000db10af3c x1846780607116416/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233662 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233637.659785:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607116416/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233662 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233637.659793:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233637.659795:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233637.659797:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116416:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233637.659802:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116480:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233637.659819:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233637/real 1761233637] req@00000000b72feabb x1846780607116480/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233662 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233637.659880:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607116480/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233662 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233637.659893:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233637.659898:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233637.659902:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116480:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233638.107209:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233639.131004:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233640.155103:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233641.179139:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233642.203117:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233642.779024:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233642.779034:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233642.779038:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233642.779047:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6515 00000100:00080000:3.0:1761233642.779051:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233642.779057:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233642.779093:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607116544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233642.779119:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233642.779122:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233642.779124:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233642.779127:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6515 00000100:00080000:3.0:1761233642.779129:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233642.779131:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233642.779138:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233642.779146:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607116608/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233642.779153:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233642.779175:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233642.779177:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233642.779180:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6515 00000100:00080000:3.0:1761233642.779183:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233642.779185:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233642.779190:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233642.779201:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607116672/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233642.779209:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233642.779212:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233642.779213:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233642.779217:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6515 00000100:00080000:3.0:1761233642.779219:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233642.779224:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233642.779228:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233642.779246:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607116736/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233642.779266:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607116544:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233642.779429:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233642/real 1761233642] req@00000000daa39bb1 x1846780607116544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233667 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233642.779444:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607116544/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233667 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233642.779453:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233642.779456:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233642.779459:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607116544:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233642.779464:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116608:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233642.779483:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233642/real 1761233642] req@000000003ad149cc x1846780607116608/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233667 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233642.779491:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607116608/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233667 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233642.779497:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233642.779498:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233642.779500:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116608:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233642.779504:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116672:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233642.779518:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233642/real 1761233642] req@0000000088473b3e x1846780607116672/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233667 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233642.779532:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607116672/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233667 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233642.779538:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233642.779539:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233642.779553:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116672:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233642.779581:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116736:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233642.779610:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233642/real 1761233642] req@000000004e3f612a x1846780607116736/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233667 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233642.779619:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607116736/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233667 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233642.779625:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233642.779626:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233642.779628:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116736:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233643.227032:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233644.251001:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233645.275006:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233646.299085:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233647.323187:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233647.898946:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233647.898958:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233647.898961:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233647.898972:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6520 00000100:00080000:3.0:1761233647.898975:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233647.898982:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233647.899019:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e3f612a x1846780607116800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233647.899046:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233647.899050:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233647.899052:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233647.899056:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6520 00000100:00080000:3.0:1761233647.899058:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233647.899062:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233647.899070:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233647.899083:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088473b3e x1846780607116864/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233647.899094:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761233647.899095:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607116800:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:3.0:1761233647.899111:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233647.899115:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233647.899123:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6520 00000100:00080000:3.0:1761233647.899126:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233647.899131:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233647.899136:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233647.899149:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003ad149cc x1846780607116928/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233647.899158:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233647.899162:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233647.899164:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233647.899168:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6520 00000100:00080000:3.0:1761233647.899170:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233647.899175:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233647.899179:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233647.899183:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233647/real 1761233647] req@000000004e3f612a x1846780607116800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233672 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:3.0:1761233647.899192:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daa39bb1 x1846780607116992/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233647.899207:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004e3f612a x1846780607116800/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233672 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233647.899219:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233647.899221:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233647.899226:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e3f612a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607116800:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233647.899250:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116864:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233647.899312:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233647/real 1761233647] req@0000000088473b3e x1846780607116864/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233672 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233647.899324:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000088473b3e x1846780607116864/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233672 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233647.899333:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233647.899335:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233647.899338:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088473b3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116864:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233647.899344:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116928:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233647.899370:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233647/real 1761233647] req@000000003ad149cc x1846780607116928/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233672 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233647.899386:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003ad149cc x1846780607116928/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233672 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233647.899397:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233647.899400:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233647.899405:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ad149cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116928:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233647.899414:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116992:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233647.899435:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233647/real 1761233647] req@00000000daa39bb1 x1846780607116992/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233672 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233647.899445:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000daa39bb1 x1846780607116992/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233672 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233647.899453:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233647.899455:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233647.899458:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daa39bb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607116992:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233648.347122:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233649.371143:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233650.395047:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233651.419115:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233652.443089:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233653.018925:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233653.018934:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233653.018937:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233653.018947:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6525 00000100:00080000:3.0:1761233653.018949:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233653.018955:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233653.018987:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a9bdefd x1846780607117056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233653.019008:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233653.019010:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233653.019012:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233653.019014:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6525 00000100:00080000:3.0:1761233653.019016:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233653.019019:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233653.019025:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233653.019033:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000018a8f5da x1846780607117120/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233653.019039:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233653.019041:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233653.019042:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233653.019045:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6525 00000100:00080000:3.0:1761233653.019046:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233653.019048:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233653.019051:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233653.019059:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd684ccc x1846780607117184/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233653.019067:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607117056:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:3.0:1761233653.019069:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233653.019080:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233653.019083:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233653.019085:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6525 00000100:00080000:3.0:1761233653.019087:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233653.019088:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233653.019091:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233653.019101:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086ccb527 x1846780607117248/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233653.019181:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117120:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233653.019245:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117184:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233653.019279:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117248:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233653.467059:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233654.491044:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233655.515136:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761233656.091250:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233653/real 1761233656] req@00000000bd684ccc x1846780607117184/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233678 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233656.091280:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000bd684ccc x1846780607117184/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233678 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233656.091292:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233656.091296:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233656.091301:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd684ccc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117184:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233656.091307:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233653/real 1761233656] req@0000000086ccb527 x1846780607117248/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233678 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233656.091316:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086ccb527 x1846780607117248/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233678 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233656.091324:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233656.091326:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233656.091330:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086ccb527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117248:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233656.091335:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233653/real 1761233656] req@000000005a9bdefd x1846780607117056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233678 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233656.091344:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a9bdefd x1846780607117056/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233678 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233656.091350:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233656.091352:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233656.091355:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a9bdefd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607117056:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233656.091360:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233653/real 1761233656] req@0000000018a8f5da x1846780607117120/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233678 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233656.091369:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000018a8f5da x1846780607117120/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233678 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233656.091375:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233656.091378:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233656.091380:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018a8f5da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117120:192.168.201.104@tcp:38:kworker/u8:1.0 00010000:00010000:3.0:1761233656.539058:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233657.563095:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233658.139040:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233658.139053:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233658.139057:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233658.139070:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6530 00000100:00080000:1.0:1761233658.139073:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233658.139080:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233658.139133:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607117312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233658.139162:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233658.139165:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233658.139167:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233658.139170:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6530 00000100:00080000:1.0:1761233658.139172:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233658.139174:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233658.139183:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233658.139192:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607117376/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233658.139199:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233658.139201:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233658.139203:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233658.139205:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6530 00000100:00080000:1.0:1761233658.139206:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233658.139208:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233658.139212:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233658.139215:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607117312:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:1.0:1761233658.139220:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607117440/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233658.139227:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233658.139230:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233658.139231:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233658.139233:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6530 00000100:00080000:1.0:1761233658.139234:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233658.139236:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233658.139239:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233658.139246:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607117504/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233658.139340:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233658/real 1761233658] req@00000000b72feabb x1846780607117312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233683 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233658.139361:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607117312/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233683 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233658.139373:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233658.139376:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233658.139381:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607117312:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233658.139388:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117376:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233658.139416:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233658/real 1761233658] req@00000000db10af3c x1846780607117376/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233683 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233658.139434:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607117376/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233683 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233658.139442:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233658.139444:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233658.139448:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117376:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233658.139488:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117440:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233658.139518:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233658/real 1761233658] req@00000000d07717c4 x1846780607117440/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233683 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233658.139530:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607117440/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233683 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233658.139539:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233658.139542:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233658.139545:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117440:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233658.139551:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117504:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233658.139573:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233658/real 1761233658] req@00000000659d78e3 x1846780607117504/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233683 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233658.139585:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607117504/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233683 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233658.139594:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233658.139597:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233658.139600:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117504:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233658.587053:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233659.611101:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233660.635059:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233661.659146:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233662.683021:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233663.258989:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233663.259001:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233663.259005:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233663.259015:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6535 00000100:00080000:3.0:1761233663.259019:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233663.259026:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233663.259107:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031e043f4 x1846780607117568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233663.259149:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233663.259153:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233663.259154:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233663.259157:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6535 00000100:00080000:3.0:1761233663.259159:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233663.259162:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233663.259170:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233663.259180:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7e4e432 x1846780607117632/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233663.259189:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233663.259194:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233663.259196:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233663.259198:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6535 00000100:00080000:3.0:1761233663.259200:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233663.259202:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233663.259206:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233663.259214:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f87d0618 x1846780607117696/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233663.259228:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607117568:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761233663.259228:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233663.259237:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233663.259239:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233663.259249:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6535 00000100:00080000:3.0:1761233663.259251:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233663.259253:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233663.259256:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233663.259264:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007361665d x1846780607117760/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233663.259366:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233663/real 1761233663] req@0000000031e043f4 x1846780607117568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233688 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233663.259389:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000031e043f4 x1846780607117568/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233688 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233663.259401:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233663.259405:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233663.259410:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031e043f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607117568:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:2.0:1761233663.259419:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117632:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233663.259456:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233663/real 1761233663] req@00000000c7e4e432 x1846780607117632/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233688 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233663.259467:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c7e4e432 x1846780607117632/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233688 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233663.259475:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233663.259478:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233663.259481:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7e4e432 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117632:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:2.0:1761233663.259527:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117696:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233663.259552:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233663/real 1761233663] req@00000000f87d0618 x1846780607117696/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233688 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233663.259563:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f87d0618 x1846780607117696/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233688 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233663.259573:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233663.259576:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233663.259579:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f87d0618 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117696:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233663.259585:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117760:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761233663.259605:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233663/real 1761233663] req@000000007361665d x1846780607117760/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233688 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:2.0:1761233663.259617:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007361665d x1846780607117760/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233688 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:2.0:1761233663.259625:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233663.259628:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233663.259631:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007361665d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117760:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233663.707006:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233664.731067:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233665.754976:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233666.779051:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233667.803038:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233668.379002:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233668.379017:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233668.379023:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233668.379037:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6540 00000100:00080000:1.0:1761233668.379041:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233668.379050:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233668.379105:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607117824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233668.379151:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233668.379155:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233668.379156:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233668.379161:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6540 00000100:00080000:1.0:1761233668.379163:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233668.379167:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233668.379178:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233668.379191:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607117888/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233668.379202:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607117824:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761233668.379204:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233668.379211:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233668.379213:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233668.379218:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6540 00000100:00080000:1.0:1761233668.379220:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233668.379223:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233668.379230:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233668.379243:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607117952/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233668.379251:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233668.379255:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233668.379256:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233668.379260:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6540 00000100:00080000:1.0:1761233668.379262:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233668.379266:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233668.379271:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233668.379282:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607118016/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233668.379310:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233668/real 1761233668] req@00000000659d78e3 x1846780607117824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233693 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233668.379326:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607117824/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233693 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233668.379334:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233668.379337:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233668.379340:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607117824:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233668.379362:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117888:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233668.379378:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233668/real 1761233668] req@00000000d07717c4 x1846780607117888/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233693 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233668.379384:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607117888/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233693 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233668.379389:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233668.379390:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233668.379391:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117888:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233668.379395:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117952:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233668.379407:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233668/real 1761233668] req@00000000db10af3c x1846780607117952/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233693 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233668.379413:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607117952/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233693 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233668.379418:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233668.379419:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233668.379420:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607117952:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233668.379423:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118016:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233668.379433:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233668/real 1761233668] req@00000000b72feabb x1846780607118016/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233693 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233668.379444:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607118016/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233693 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233668.379449:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233668.379450:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233668.379451:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118016:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233668.827079:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233669.851012:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233670.875051:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233671.899045:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233672.923064:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233673.498951:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233673.498960:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233673.498963:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233673.498972:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6545 00000100:00080000:0.0:1761233673.498975:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233673.498981:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233673.499010:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607118080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233673.499029:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233673.499032:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233673.499033:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233673.499036:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6545 00000100:00080000:0.0:1761233673.499038:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233673.499040:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233673.499046:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233673.499054:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607118144/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233673.499061:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233673.499064:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1761233673.499064:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607118080:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761233673.499066:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233673.499072:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6545 00000100:00080000:0.0:1761233673.499074:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233673.499076:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233673.499079:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233673.499087:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607118208/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233673.499092:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233673.499094:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233673.499096:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233673.499098:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6545 00000100:00080000:0.0:1761233673.499100:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233673.499102:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233673.499104:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233673.499111:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607118272/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233673.499143:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233673/real 1761233673] req@0000000023ab3a3f x1846780607118080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233698 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233673.499156:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607118080/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233698 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233673.499163:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233673.499165:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233673.499167:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607118080:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233673.499186:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118144:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233673.499202:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233673/real 1761233673] req@0000000001b2f813 x1846780607118144/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233698 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233673.499208:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607118144/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233698 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233673.499212:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233673.499213:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233673.499215:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118144:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233673.499219:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118208:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233673.499230:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233673/real 1761233673] req@00000000275aeee5 x1846780607118208/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233698 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233673.499236:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607118208/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233698 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233673.499241:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233673.499242:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233673.499244:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118208:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233673.499247:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118272:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233673.499257:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233673/real 1761233673] req@000000000243251f x1846780607118272/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233698 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233673.499263:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607118272/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233698 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233673.499267:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233673.499269:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233673.499270:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118272:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233673.947057:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233674.971066:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233675.994994:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233677.019031:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233678.043017:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233678.618945:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233678.618955:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233678.618957:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233678.618966:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6551 00000100:00080000:0.0:1761233678.618969:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233678.618974:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233678.619008:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8f2be78 x1846780607118336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233678.619036:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233678.619040:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233678.619042:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233678.619046:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6551 00000100:00080000:0.0:1761233678.619048:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233678.619051:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233678.619060:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233678.619073:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcce404c x1846780607118400/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233678.619080:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607118336:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761233678.619087:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233678.619093:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233678.619094:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233678.619099:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6551 00000100:00080000:0.0:1761233678.619101:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233678.619105:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233678.619110:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233678.619125:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f60e210 x1846780607118464/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233678.619132:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233678.619134:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233678.619135:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233678.619138:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6551 00000100:00080000:0.0:1761233678.619139:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233678.619141:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233678.619144:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233678.619152:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000243251f x1846780607118528/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233678.619165:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233678/real 1761233678] req@00000000b8f2be78 x1846780607118336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233703 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233678.619179:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b8f2be78 x1846780607118336/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233703 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233678.619187:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233678.619189:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233678.619192:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8f2be78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607118336:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233678.619210:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118400:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233678.619228:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233678/real 1761233678] req@00000000fcce404c x1846780607118400/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233703 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233678.619234:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcce404c x1846780607118400/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233703 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233678.619239:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233678.619240:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233678.619242:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcce404c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118400:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233678.619245:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118464:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233678.619256:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233678/real 1761233678] req@000000007f60e210 x1846780607118464/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233703 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233678.619263:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007f60e210 x1846780607118464/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233703 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233678.619270:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233678.619271:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233678.619273:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f60e210 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118464:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233678.619277:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118528:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233678.619292:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233678/real 1761233678] req@000000000243251f x1846780607118528/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233703 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233678.619298:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000243251f x1846780607118528/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233703 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233678.619302:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233678.619303:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233678.619305:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000243251f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118528:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233679.067088:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233680.090987:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233681.115048:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233682.138991:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233683.163018:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233683.738984:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233683.738997:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233683.739000:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233683.739009:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6556 00000100:00080000:2.0:1761233683.739012:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233683.739018:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233683.739048:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000091106a97 x1846780607118592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233683.739070:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233683.739072:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233683.739074:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233683.739076:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6556 00000100:00080000:2.0:1761233683.739077:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233683.739079:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233683.739085:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233683.739093:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094135f44 x1846780607118656/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233683.739101:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233683.739103:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233683.739105:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761233683.739106:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607118592:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761233683.739110:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6556 00000100:00080000:2.0:1761233683.739113:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233683.739115:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233683.739118:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233683.739127:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607118720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233683.739132:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233683.739134:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233683.739135:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233683.739138:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6556 00000100:00080000:2.0:1761233683.739139:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233683.739141:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233683.739143:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233683.739150:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607118784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233683.739229:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233683/real 1761233683] req@0000000091106a97 x1846780607118592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233708 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233683.739241:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000091106a97 x1846780607118592/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233708 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233683.739248:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233683.739250:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233683.739253:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091106a97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607118592:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233683.739271:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118656:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233683.739289:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233683/real 1761233683] req@0000000094135f44 x1846780607118656/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233708 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233683.739296:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000094135f44 x1846780607118656/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233708 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233683.739300:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233683.739302:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233683.739303:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094135f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118656:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233683.739306:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118720:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233683.739318:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233683/real 1761233683] req@0000000064978143 x1846780607118720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233708 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233683.739324:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607118720/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233708 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233683.739329:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233683.739330:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233683.739331:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118720:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233683.739334:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118784:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233683.739346:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233683/real 1761233683] req@00000000c069b612 x1846780607118784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233708 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233683.739352:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607118784/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233708 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233683.739357:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233683.739358:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233683.739359:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118784:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233684.187036:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233685.211006:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233686.234972:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233687.258992:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233688.283020:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233688.858916:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233688.858928:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233688.858933:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233688.858944:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6561 00000100:00080000:3.0:1761233688.858948:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233688.858955:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233688.858990:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000090b40c66 x1846780607118848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233688.859013:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233688.859015:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233688.859016:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233688.859020:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6561 00000100:00080000:3.0:1761233688.859021:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233688.859024:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233688.859031:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233688.859038:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000083f16a6b x1846780607118912/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233688.859044:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233688.859046:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233688.859047:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233688.859049:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6561 00000100:00080000:3.0:1761233688.859051:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233688.859053:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233688.859056:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233688.859064:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a58ca11e x1846780607118976/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233688.859071:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607118848:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761233688.859077:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233688.859081:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233688.859082:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233688.859086:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6561 00000100:00080000:3.0:1761233688.859089:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233688.859093:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233688.859098:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233688.859111:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f8e688e x1846780607119040/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233688.859158:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233688/real 1761233688] req@0000000090b40c66 x1846780607118848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233713 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233688.859176:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000090b40c66 x1846780607118848/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233713 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233688.859183:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233688.859185:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233688.859187:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090b40c66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607118848:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233688.859192:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118912:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233688.859215:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233688/real 1761233688] req@0000000083f16a6b x1846780607118912/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233713 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233688.859225:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000083f16a6b x1846780607118912/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233713 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233688.859233:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233688.859235:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233688.859238:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000083f16a6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118912:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233688.859263:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118976:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233688.859287:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233688/real 1761233688] req@00000000a58ca11e x1846780607118976/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233713 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233688.859296:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a58ca11e x1846780607118976/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233713 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233688.859304:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233688.859305:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233688.859308:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a58ca11e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607118976:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233688.859314:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119040:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233688.859347:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233688/real 1761233688] req@000000001f8e688e x1846780607119040/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233713 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233688.859359:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000001f8e688e x1846780607119040/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233713 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233688.859365:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233688.859366:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233688.859368:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f8e688e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119040:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233689.307023:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233690.330987:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233691.355010:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233692.378992:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233693.403026:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233693.978952:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233693.978960:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233693.978963:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233693.978972:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6566 00000100:00080000:3.0:1761233693.978975:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233693.978981:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233693.979014:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607119104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233693.979036:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233693.979039:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233693.979040:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233693.979043:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6566 00000100:00080000:3.0:1761233693.979044:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233693.979046:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233693.979052:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233693.979059:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607119168/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233693.979064:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233693.979066:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233693.979067:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233693.979069:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6566 00000100:00080000:3.0:1761233693.979070:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233693.979073:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233693.979076:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233693.979082:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607119232/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233693.979087:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233693.979089:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1761233693.979090:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233693.979092:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6566 00000100:00080000:3.0:1761233693.979093:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233693.979095:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233693.979097:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233693.979102:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607119296/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233693.979122:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607119104:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233693.979195:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233693/real 1761233693] req@00000000fc01c41c x1846780607119104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233718 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233693.979210:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607119104/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233718 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233693.979217:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233693.979218:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233693.979221:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607119104:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233693.979225:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119168:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233693.979239:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233693/real 1761233693] req@00000000408f088c x1846780607119168/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233718 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233693.979252:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607119168/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233718 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233693.979256:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233693.979258:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233693.979259:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119168:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233693.979262:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119232:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233693.979275:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233693/real 1761233693] req@000000002ce934d3 x1846780607119232/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233718 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233693.979281:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607119232/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233718 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233693.979286:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233693.979287:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233693.979288:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119232:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233693.979292:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119296:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233693.979303:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233693/real 1761233693] req@000000006572b591 x1846780607119296/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233718 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233693.979308:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607119296/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233718 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233693.979313:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233693.979314:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233693.979316:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119296:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233694.427041:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233695.451027:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233696.475011:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233697.499000:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233698.523045:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233699.098932:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233699.098942:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233699.098945:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233699.098954:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6571 00000100:00080000:1.0:1761233699.098957:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233699.098963:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233699.098997:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607119360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233699.099020:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233699.099023:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233699.099024:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233699.099027:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6571 00000100:00080000:1.0:1761233699.099028:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233699.099030:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233699.099037:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233699.099045:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607119424/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233699.099051:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233699.099053:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233699.099054:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233699.099057:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6571 00000100:00080000:1.0:1761233699.099058:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233699.099062:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233699.099065:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233699.099072:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607119488/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233699.099077:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233699.099079:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233699.099080:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233699.099083:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6571 00000100:00080000:1.0:1761233699.099084:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233699.099086:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233699.099089:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233699.099093:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607119552/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233699.099105:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607119360:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233699.099223:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233699/real 1761233699] req@00000000b72feabb x1846780607119360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233724 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233699.099240:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607119360/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233724 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233699.099250:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233699.099252:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233699.099255:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607119360:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233699.099261:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119424:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233699.099278:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233699/real 1761233699] req@00000000db10af3c x1846780607119424/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233724 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233699.099285:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607119424/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233724 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233699.099289:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233699.099291:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233699.099292:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119424:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233699.099296:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119488:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233699.099308:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233699/real 1761233699] req@00000000d07717c4 x1846780607119488/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233724 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233699.099315:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607119488/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233724 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233699.099320:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233699.099321:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233699.099322:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119488:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233699.099348:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119552:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233699.099361:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233699/real 1761233699] req@00000000659d78e3 x1846780607119552/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233724 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233699.099367:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607119552/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233724 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233699.099372:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233699.099373:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233699.099375:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119552:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233699.547044:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233700.571042:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233701.594979:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233702.619011:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233703.643047:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233704.218942:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233704.218955:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233704.218959:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233704.218970:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6576 00000100:00080000:1.0:1761233704.218975:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233704.218982:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233704.219030:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607119616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233704.219057:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233704.219060:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233704.219061:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233704.219065:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6576 00000100:00080000:1.0:1761233704.219066:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233704.219068:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233704.219075:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233704.219083:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607119680/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233704.219090:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233704.219092:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233704.219094:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233704.219098:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6576 00000100:00080000:1.0:1761233704.219100:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233704.219102:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233704.219105:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233704.219111:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607119744/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233704.219117:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233704.219121:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233704.219123:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233704.219130:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6576 00000100:00100000:2.0:1761233704.219132:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607119616:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761233704.219135:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233704.219141:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233704.219147:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233704.219154:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607119808/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233704.219233:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233704/real 1761233704] req@00000000659d78e3 x1846780607119616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233729 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233704.219254:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607119616/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233729 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233704.219264:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233704.219268:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233704.219284:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607119616:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233704.219292:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119680:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233704.219323:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233704/real 1761233704] req@00000000d07717c4 x1846780607119680/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233729 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233704.219335:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607119680/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233729 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233704.219343:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233704.219346:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233704.219349:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119680:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233704.219375:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119744:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233704.219446:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233704/real 1761233704] req@00000000db10af3c x1846780607119744/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233729 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233704.219457:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607119744/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233729 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233704.219466:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233704.219468:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233704.219471:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119744:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233704.219476:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119808:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233704.219494:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233704/real 1761233704] req@00000000b72feabb x1846780607119808/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233729 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233704.219505:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607119808/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233729 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233704.219513:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233704.219515:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233704.219517:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119808:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233704.667070:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233705.691072:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233706.715042:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233707.739088:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233708.762997:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761233709.339004:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233709.339014:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:2.0:1761233709.339017:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233709.339027:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6581 00000100:00080000:2.0:1761233709.339029:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233709.339035:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:2.0:1761233709.339064:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f38f27df x1846780607119872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233709.339088:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233709.339091:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1761233709.339092:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233709.339094:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6581 00000100:00080000:2.0:1761233709.339096:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233709.339098:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233709.339105:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761233709.339113:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017386078 x1846780607119936/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233709.339120:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233709.339122:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761233709.339124:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761233709.339125:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607119872:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:2.0:1761233709.339131:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6581 00000100:00080000:2.0:1761233709.339134:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233709.339136:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233709.339140:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233709.339151:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c069b612 x1846780607120000/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761233709.339156:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761233709.339159:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1761233709.339160:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761233709.339163:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6581 00000100:00080000:2.0:1761233709.339164:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761233709.339166:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:2.0:1761233709.339169:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1761233709.339175:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000064978143 x1846780607120064/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233709.339211:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233709/real 1761233709] req@00000000f38f27df x1846780607119872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233734 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233709.339240:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f38f27df x1846780607119872/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233734 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233709.339252:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233709.339255:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233709.339260:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f38f27df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607119872:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233709.339285:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119936:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233709.339360:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233709/real 1761233709] req@0000000017386078 x1846780607119936/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233734 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233709.339368:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000017386078 x1846780607119936/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233734 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233709.339374:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233709.339376:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233709.339377:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017386078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607119936:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233709.339382:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120000:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233709.339408:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233709/real 1761233709] req@00000000c069b612 x1846780607120000/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233734 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233709.339418:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c069b612 x1846780607120000/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233734 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233709.339427:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233709.339429:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233709.339442:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c069b612 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120000:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233709.339448:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120064:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233709.339471:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233709/real 1761233709] req@0000000064978143 x1846780607120064/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233734 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233709.339482:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000064978143 x1846780607120064/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233734 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233709.339491:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233709.339493:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233709.339495:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064978143 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120064:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233709.787157:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233710.811037:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761233711.835003:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233712.859051:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233713.883039:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233714.458965:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233714.458977:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:3.0:1761233714.458981:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233714.458992:0:11073:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6586 00000100:00080000:3.0:1761233714.458995:0:11073:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233714.459002:0:11073:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:3.0:1761233714.459038:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006572b591 x1846780607120128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233714.459067:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233714.459071:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1761233714.459072:0:11073:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233714.459076:0:11073:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6586 00000100:00080000:3.0:1761233714.459078:0:11073:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233714.459081:0:11073:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233714.459091:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1761233714.459103:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ce934d3 x1846780607120192/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233714.459112:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233714.459115:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761233714.459117:0:11073:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233714.459123:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6586 00000100:00080000:3.0:1761233714.459127:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233714.459131:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233714.459136:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233714.459147:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408f088c x1846780607120256/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761233714.459157:0:11073:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761233714.459164:0:11073:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1761233714.459165:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607120128:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00080000:3.0:1761233714.459168:0:11073:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761233714.459179:0:11073:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6586 00000100:00080000:3.0:1761233714.459182:0:11073:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761233714.459186:0:11073:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:3.0:1761233714.459191:0:11073:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1761233714.459203:0:11073:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc01c41c x1846780607120320/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761233714.459312:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233714/real 1761233714] req@000000006572b591 x1846780607120128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233739 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233714.459332:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006572b591 x1846780607120128/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233739 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233714.459343:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233714.459348:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233714.459352:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006572b591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607120128:192.168.201.104@tcp:250:kworker/u8:2.0 00000100:00100000:1.0:1761233714.459360:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120192:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233714.459386:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233714/real 1761233714] req@000000002ce934d3 x1846780607120192/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233739 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233714.459396:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ce934d3 x1846780607120192/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233739 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233714.459402:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233714.459404:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233714.459407:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ce934d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120192:192.168.201.104@tcp:38:kworker/u8:2.0 00000100:00100000:1.0:1761233714.459445:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120256:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233714.459468:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233714/real 1761233714] req@00000000408f088c x1846780607120256/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233739 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233714.459479:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000408f088c x1846780607120256/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233739 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233714.459486:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233714.459489:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233714.459492:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408f088c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120256:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233714.459498:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120320:192.168.201.104@tcp:8:kworker/u8:2.0 00000100:00100000:1.0:1761233714.459520:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233714/real 1761233714] req@00000000fc01c41c x1846780607120320/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233739 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00100000:1.0:1761233714.459529:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fc01c41c x1846780607120320/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233739 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:2.0' 00000100:00080000:1.0:1761233714.459536:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233714.459539:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:1.0:1761233714.459541:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc01c41c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120320:192.168.201.104@tcp:8:kworker/u8:2.0 00010000:00010000:3.0:1761233714.907070:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233715.931025:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233716.955011:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233717.979028:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233719.002972:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233719.578970:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233719.578979:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233719.578982:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233719.578991:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6591 00000100:00080000:1.0:1761233719.578994:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233719.578999:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233719.579034:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607120384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233719.579056:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233719.579059:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233719.579060:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233719.579062:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6591 00000100:00080000:1.0:1761233719.579063:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233719.579065:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233719.579072:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233719.579079:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607120448/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233719.579085:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233719.579088:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233719.579089:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233719.579091:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6591 00000100:00080000:1.0:1761233719.579092:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233719.579094:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233719.579097:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233719.579103:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607120512/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233719.579108:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233719.579110:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233719.579112:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761233719.579115:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607120384:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761233719.579121:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6591 00000100:00080000:1.0:1761233719.579124:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233719.579126:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233719.579129:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233719.579135:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607120576/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233719.579210:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120448:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233719.579247:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120512:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233719.579271:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120576:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233720.027072:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233721.051019:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233722.075062:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761233722.651214:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233719/real 1761233722] req@00000000d07717c4 x1846780607120512/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233744 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233722.651250:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607120512/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233744 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233722.651261:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233722.651265:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233722.651268:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120512:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233722.651274:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233719/real 1761233722] req@00000000659d78e3 x1846780607120576/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233744 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233722.651283:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607120576/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233744 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233722.651289:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233722.651290:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233722.651292:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120576:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233722.651296:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233719/real 1761233722] req@00000000b72feabb x1846780607120384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233744 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233722.651303:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607120384/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233744 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233722.651307:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233722.651309:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233722.651310:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607120384:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233722.651314:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233719/real 1761233722] req@00000000db10af3c x1846780607120448/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233744 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233722.651320:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607120448/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233744 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233722.651324:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233722.651326:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233722.651327:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120448:192.168.201.104@tcp:38:kworker/u8:1.0 00010000:00010000:3.0:1761233723.099061:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233724.123028:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233724.699008:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233724.699021:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:0.0:1761233724.699024:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233724.699036:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6597 00000100:00080000:0.0:1761233724.699039:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233724.699046:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:0.0:1761233724.699092:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275aeee5 x1846780607120640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233724.699116:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233724.699119:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1761233724.699120:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233724.699123:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6597 00000100:00080000:0.0:1761233724.699124:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233724.699127:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233724.699134:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1761233724.699142:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001b2f813 x1846780607120704/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761233724.699149:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233724.699151:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761233724.699152:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233724.699155:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6597 00000100:00080000:0.0:1761233724.699156:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233724.699159:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233724.699162:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233724.699172:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023ab3a3f x1846780607120768/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233724.699177:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607120640:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:0.0:1761233724.699185:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761233724.699190:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1761233724.699191:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761233724.699195:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6597 00000100:00080000:0.0:1761233724.699197:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761233724.699202:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:0.0:1761233724.699206:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1761233724.699217:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a096598 x1846780607120832/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233724.699272:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233724/real 1761233724] req@00000000275aeee5 x1846780607120640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233749 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233724.699291:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000275aeee5 x1846780607120640/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233749 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233724.699303:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233724.699306:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233724.699310:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275aeee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607120640:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233724.699323:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120704:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233724.699358:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233724/real 1761233724] req@0000000001b2f813 x1846780607120704/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233749 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233724.699371:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000001b2f813 x1846780607120704/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233749 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233724.699379:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233724.699382:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233724.699385:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001b2f813 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120704:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233724.699413:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120768:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233724.699443:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233724/real 1761233724] req@0000000023ab3a3f x1846780607120768/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233749 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233724.699454:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000023ab3a3f x1846780607120768/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233749 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233724.699461:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233724.699463:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233724.699466:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023ab3a3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120768:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233724.699471:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120832:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233724.699492:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233724/real 1761233724] req@000000005a096598 x1846780607120832/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233749 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233724.699502:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005a096598 x1846780607120832/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233749 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233724.699509:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233724.699512:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233724.699514:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a096598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120832:192.168.201.104@tcp:8:kworker/u8:1.0 00010000:00010000:3.0:1761233725.147003:0:10238:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233726.170964:0:9093:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233727.194986:0:5009:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233728.219004:0:5010:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761233729.242945:0:9102:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233729.818929:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 68b5ba96-fdab-44f9-b92e-5c61bdabcbf7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233729.818939:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00080000:1.0:1761233729.818941:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233729.818950:0:10992:0:(import.c:534:import_select_connection()) MGC192.168.201.104@tcp: connect to NID 192.168.201.104@tcp last attempt 6602 00000100:00080000:1.0:1761233729.818953:0:10992:0:(import.c:581:import_select_connection()) MGC192.168.201.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233729.818958:0:10992:0:(import.c:615:import_select_connection()) MGC192.168.201.104@tcp: import 00000000ba87f1bd using connection MGC192.168.201.104@tcp_0/192.168.201.104@tcp 00000100:00100000:1.0:1761233729.819014:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db10af3c x1846780607120896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233729.819041:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233729.819045:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1761233729.819047:0:10992:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233729.819052:0:10992:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6602 00000100:00080000:1.0:1761233729.819054:0:10992:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233729.819057:0:10992:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffff9001d2597000: import 000000002dbdfc93 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233729.819066:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1761233729.819079:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b72feabb x1846780607120960/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233729.819089:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233729.819092:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761233729.819093:0:10992:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761233729.819098:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607120896:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00080000:1.0:1761233729.819101:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6602 00000100:00080000:1.0:1761233729.819105:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233729.819110:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffff9001d2597000: import 00000000a1f01947 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233729.819115:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233729.819127:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000659d78e3 x1846780607121024/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761233729.819135:0:10992:0:(pinger.c:251:ptlrpc_pinger_process_import()) 0995b904-d672-4cf9-8bdd-5554fffeedd5->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761233729.819138:0:10992:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1761233729.819139:0:10992:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761233729.819143:0:10992:0:(import.c:534:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: connect to NID 192.168.201.104@tcp last attempt 6602 00000100:00080000:1.0:1761233729.819145:0:10992:0:(import.c:581:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761233729.819148:0:10992:0:(import.c:615:import_select_connection()) lustre-OST0001-osc-ffff9001d2597000: import 000000008bcc4531 using connection 192.168.201.104@tcp/192.168.201.104@tcp 00000008:00100000:1.0:1761233729.819151:0:10992:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1761233729.819160:0:10992:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d07717c4 x1846780607121088/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761233729.819202:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233729/real 1761233729] req@00000000db10af3c x1846780607120896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233754 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233729.819220:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db10af3c x1846780607120896/t0(0) o250->MGC192.168.201.104@tcp@192.168.201.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233754 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233729.819229:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000ba87f1bd MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233729.819233:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233729.819237:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db10af3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:68b5ba96-fdab-44f9-b92e-5c61bdabcbf7:2227:1846780607120896:192.168.201.104@tcp:250:kworker/u8:1.0 00000100:00100000:2.0:1761233729.819258:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120960:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233729.819284:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233729/real 1761233729] req@00000000b72feabb x1846780607120960/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233754 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233729.819293:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000b72feabb x1846780607120960/t0(0) o38->lustre-MDT0000-mdc-ffff9001d2597000@192.168.201.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233754 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233729.819306:0:2227:0:(import.c:86:import_set_state_nolock()) 000000002dbdfc93 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233729.819308:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233729.819311:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b72feabb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607120960:192.168.201.104@tcp:38:kworker/u8:1.0 00000100:00100000:2.0:1761233729.819316:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607121024:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233729.819339:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233729/real 1761233729] req@00000000659d78e3 x1846780607121024/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233754 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233729.819349:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000659d78e3 x1846780607121024/t0(0) o8->lustre-OST0000-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233754 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233729.819356:0:2227:0:(import.c:86:import_set_state_nolock()) 00000000a1f01947 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233729.819358:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233729.819360:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000659d78e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607121024:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233729.819365:0:2227:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607121088:192.168.201.104@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761233729.819388:0:2227:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233729/real 1761233729] req@00000000d07717c4 x1846780607121088/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233754 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00100000:2.0:1761233729.819398:0:2227:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d07717c4 x1846780607121088/t0(0) o8->lustre-OST0001-osc-ffff9001d2597000@192.168.201.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233754 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'kworker/u8:1.0' 00000100:00080000:2.0:1761233729.819405:0:2227:0:(import.c:86:import_set_state_nolock()) 000000008bcc4531 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761233729.819407:0:2227:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.104@tcp failed (-110) 00000100:00100000:2.0:1761233729.819410:0:2227:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d07717c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0995b904-d672-4cf9-8bdd-5554fffeedd5:2227:1846780607121088:192.168.201.104@tcp:8:kworker/u8:1.0 Debug log: 30625 lines, 30625 kept, 0 dropped, 0 bad.